newrelic_security 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b1160c4b821177b6ce0400848a72bf09899780ff83abadb60b3e60f35b8338b
-  data.tar.gz: 98cfb5a7d513e842690dce03aa515a7e148a252a5f0666f16e40899e2ad511d0
+  metadata.gz: 82cddb6a77c7bac2c46e31b5e0577dd346aa0e78b919fc8fb3c2765a7192c0a6
+  data.tar.gz: cb8f47120816909cb30ae0d16f9c6c4da3f684a8497747ccab3bbf12aa36e501
 SHA512:
-  metadata.gz: 56dea910383ac11f0b87a29f304e0f5042336c142c1d8ca60a2cdaf15bf172102000e28e8f382fd99781afebd061a0693e3f5ff8ea9fba981cdc4de128abdc20
-  data.tar.gz: 04bb28178707141c66ac94158fdb3cd632ccb22a1a9c4b4f5f7d9438989ee3c6fba2124abc89ffbf135d34e249b5921cd5d4b924e50768f07363b219567867bb
+  metadata.gz: 96abdbedd151b564c30d2ccafc1e98b46185130d6028634877ef0a42c1dc8ff775cf5affbcb7fdd7eee305bbb5ca33a266dfd82ebd5b7f0d2f6e396687af7897
+  data.tar.gz: a69f0fcaf8a427a4ec169ed7dd33952ee88e723a89281a2e9b46841029898fdea6a21748290e2058e861629e368da84b0b6d317181248b33f88a3a33bc5049e2

data/.github/workflows/pr_ci.yml

@@ -72,6 +72,6 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           resultPath: lib/coverage_results/.last_run.json
-          failedThreshold: 70
-          failedThresholdBranch: 33
+          failedThreshold: 67
+          failedThresholdBranch: 25
 

data/CHANGELOG.md

@@ -1,10 +1,71 @@
 # New Relic Ruby Security Agent Release Notes
 
+## v0.3.0
+
+Version 0.3.0 introduces more control on IAST scanning through new configs(exclude_from_iast_scan, scan_schedule & scan_controllers) and 
+features like API inventory for gRPC server and IAST scan start related timestamps.
+
+Updated json_version: **1.2.8**
+
+- Feature: IAST scan exclusion for apis, http request parameters(header, query & body) & IAST detection categories and scan scheduling through delay, duration & cron schedule. [PR#131](https://github.com/newrelic/csec-ruby-agent/pull/131)
+
+- Feature: IAST scan request rate limit to control IAST scan request firing. [PR#132](https://github.com/newrelic/csec-ruby-agent/pull/132)
+
+- Feature: API endpoints support for gRPC server applications. [PR#143](https://github.com/newrelic/csec-ruby-agent/pull/143)
+
+- Feature: Reporting of IAST scanning application procStartTime, trafficStartedTime & scanStartTime. [PR#136](https://github.com/newrelic/csec-ruby-agent/pull/136)
+
+- Misc Chore: Optimised SSRF events parameters to send only URL in parameters. [PR#129](https://github.com/newrelic/csec-ruby-agent/pull/129)
+
+##### New security configs
+
+```yaml
+security:
+  exclude_from_iast_scan:
+    api: []
+    http_request_parameters:
+      header: []
+      query: []
+      body: []
+    iast_detection_category:
+      insecure_settings: false
+      invalid_file_access: false
+      sql_injection: false
+      nosql_injection: false
+      ldap_injection: false
+      javascript_injection: false
+      command_injection: false
+      xpath_injection: false
+      ssrf: false
+      rxss: false
+  scan_schedule:
+    delay: 0
+    duration: 0
+    schedule: ""
+    always_sample_traces: false
+  scan_controllers:
+    iast_scan_request_rate_limit: 3600
+```
+
+##### Deprecated security configs (will be removed in next major release v1.0.0)
+```yaml
+security:
+  request:
+    body_limit: 300
+  detection:
+    rci:
+      enabled: true
+    rxss:
+      enabled: true
+    deserialization:
+      enabled: true
+```
+
 ## v0.2.0
 
 Version 0.2.0 introuduces Error reporting as part of security. Any unhandled or 5xx errors in application runtime will now be visible in IAST capability UI. Updated json_version: **1.2.4**
 
-- Feature: Unhandled and 5xx error reproting [PR#134](https://github.com/newrelic/csec-ruby-agent/pull/134)
+- Feature: Unhandled and 5xx error reporting [PR#134](https://github.com/newrelic/csec-ruby-agent/pull/134)
 
 - Bugfix: Fix for API route not present in rails7 [PR#127](https://github.com/newrelic/csec-ruby-agent/pull/127)
 

data/THIRD_PARTY_NOTICES.md

@@ -34,3 +34,11 @@ Distributed under the following license(s):
 
   * [The MIT License](http://opensource.org/licenses/MIT)
 
+
+## [parse-cron](https://github.com/siebertm/parse-cron)
+
+Copyright (C) 2013 Michael Siebert
+
+Distributed under the following license(s):
+
+  * [The MIT License](http://opensource.org/licenses/MIT)

data/lib/newrelic_security/agent/agent.rb

@@ -19,13 +19,14 @@ require 'newrelic_security/agent/control/event_stats'
 require 'newrelic_security/agent/control/exit_event'
 require 'newrelic_security/agent/control/application_runtime_error'
 require 'newrelic_security/agent/control/error_reporting'
+require 'newrelic_security/agent/control/scan_scheduler'
 require 'newrelic_security/instrumentation-security/instrumentation_loader'
 
 module NewRelic::Security
   module Agent
     class Agent
 
-      attr_accessor :websocket_client, :event_processor, :iast_client, :http_request_count, :event_processed_count, :event_sent_count, :event_drop_count, :route_map, :iast_event_stats, :rasp_event_stats, :exit_event_stats, :error_reporting
+      attr_accessor :websocket_client, :event_processor, :iast_client, :http_request_count, :event_processed_count, :event_sent_count, :event_drop_count, :route_map, :iast_event_stats, :rasp_event_stats, :exit_event_stats, :error_reporting, :scan_scheduler
 
       def initialize
         NewRelic::Security::Agent.config
@@ -44,6 +45,7 @@ module NewRelic::Security
         @rasp_event_stats = NewRelic::Security::Agent::Control::EventStats.new
         @exit_event_stats = NewRelic::Security::Agent::Control::EventStats.new
         @error_reporting = NewRelic::Security::Agent::Control::ErrorReporting.new
+        @scan_scheduler = NewRelic::Security::Agent::Control::ScanScheduler.new
       end
 
       def init
@@ -59,11 +61,24 @@ module NewRelic::Security
         NewRelic::Security::Agent.logger.error "Exception in security agent init: #{exception.inspect} #{exception.backtrace}\n"
       end
 
+      def shutdown_security_agent
+        @iast_client&.fuzzQ&.clear
+        @iast_client&.completed_requests&.clear
+        @iast_client&.pending_request_ids&.clear
+        @iast_client&.iast_data_transfer_request_processor_thread&.kill
+        NewRelic::Security::Agent.config.disable_security
+        stop_websocket_client_if_open
+      end
+
       def start_websocket_client
-        NewRelic::Security::Agent::Control::WebsocketClient.instance.close(false) if NewRelic::Security::Agent::Control::WebsocketClient.instance.is_open?
+        stop_websocket_client_if_open
         @websocket_client = NewRelic::Security::Agent::Control::WebsocketClient.instance.connect
       end
 
+      def stop_websocket_client_if_open
+        NewRelic::Security::Agent::Control::WebsocketClient.instance.close(false) if NewRelic::Security::Agent::Control::WebsocketClient.instance.is_open?
+      end
+
       def start_event_processor
         @event_processor&.event_dequeue_thread&.kill 
         @event_processor&.healthcheck_thread&.kill
@@ -72,9 +87,10 @@ module NewRelic::Security
       end
 
       def start_iast_client
-        @iast_client&.iast_dequeue_threads&.each { |t| t.kill if t }
+        @iast_client&.iast_dequeue_threads&.each { |t| t&.kill }
         @iast_client&.iast_data_transfer_request_processor_thread&.kill
         @iast_client = nil
+        NewRelic::Security::Agent.logger.info "Starting IAST client now at current time: #{Time.now}"
         @iast_client = NewRelic::Security::Agent::Control::IASTClient.new
       end
 

data/lib/newrelic_security/agent/configuration/manager.rb

@@ -27,19 +27,42 @@ module NewRelic::Security
           @cache[:log_level] = ::NewRelic::Agent.config[:log_level]
           @cache[:high_security] = ::NewRelic::Agent.config[:high_security]
           @cache[:'agent.enabled'] = ::NewRelic::Agent.config[:'security.agent.enabled']
-          @cache[:enabled] = ::NewRelic::Agent.config[:'security.enabled']
+          @cache[:'security.enabled'] = ::NewRelic::Agent.config[:'security.enabled']
+          @cache[:enabled] = false
           @cache[:mode] = ::NewRelic::Agent.config[:'security.mode']
           @cache[:validator_service_url] = ::NewRelic::Agent.config[:'security.validator_service_url']
-          @cache[:'security.detection.rci.enabled'] = ::NewRelic::Agent.config[:'security.detection.rci.enabled']
-          @cache[:'security.detection.rxss.enabled'] = ::NewRelic::Agent.config[:'security.detection.rxss.enabled']
-          @cache[:'security.detection.deserialization.enabled'] = ::NewRelic::Agent.config[:'security.detection.deserialization.enabled']
+          # TODO: Remove security.detection.* & security.request.body_limit in next major release
+          @cache[:'security.detection.rci.enabled'] = ::NewRelic::Agent.config[:'security.detection.rci.enabled'].nil? ? true : ::NewRelic::Agent.config[:'security.detection.rci.enabled']
+          @cache[:'security.detection.rxss.enabled'] = ::NewRelic::Agent.config[:'security.detection.rxss.enabled'].nil? ? true : ::NewRelic::Agent.config[:'security.detection.rxss.enabled']
+          @cache[:'security.detection.deserialization.enabled'] = ::NewRelic::Agent.config[:'security.detection.deserialization.enabled'].nil? ? true : ::NewRelic::Agent.config[:'security.detection.deserialization.enabled']
+          @cache[:'security.scan_controllers.iast_scan_request_rate_limit'] = ::NewRelic::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'].to_i
           @cache[:framework] = detect_framework
           @cache[:'security.application_info.port'] = ::NewRelic::Agent.config[:'security.application_info.port'].to_i
-          @cache[:'security.request.body_limit'] = ::NewRelic::Agent.config[:'security.request.body_limit'].to_i > 0 ? ::NewRelic::Agent.config[:'security.request.body_limit'].to_i : 300
           @cache[:listen_port] = nil
+          @cache[:process_start_time] = current_time_millis # TODO: Ruby doesn't provide process start time in pure ruby implementation using agent loading time for now.
+          @cache[:traffic_start_time] = nil
+          @cache[:scan_start_time] = nil
           @cache[:app_root] = NewRelic::Security::Agent::Utils.app_root
           @cache[:jruby_objectspace_enabled] = false
-          @cache[:json_version] = :'1.2.4'
+          @cache[:json_version] = :'1.2.8'
+          @cache[:'security.exclude_from_iast_scan.api'] = convert_to_regexp_list(::NewRelic::Agent.config[:'security.exclude_from_iast_scan.api'])
+          @cache[:'security.exclude_from_iast_scan.http_request_parameters.header'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.header']
+          @cache[:'security.exclude_from_iast_scan.http_request_parameters.query'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.query']
+          @cache[:'security.exclude_from_iast_scan.http_request_parameters.body'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.body']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.insecure_settings'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.insecure_settings']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.javascript_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.javascript_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.command_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.command_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.ssrf'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ssrf']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.rxss'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.rxss']
+          @cache[:'security.scan_schedule.delay'] = ::NewRelic::Agent.config[:'security.scan_schedule.delay'].to_i
+          @cache[:'security.scan_schedule.duration'] = ::NewRelic::Agent.config[:'security.scan_schedule.duration'].to_i
+          @cache[:'security.scan_schedule.schedule'] = ::NewRelic::Agent.config[:'security.scan_schedule.schedule']
+          @cache[:'security.scan_schedule.always_sample_traces'] = ::NewRelic::Agent.config[:'security.scan_schedule.always_sample_traces']
 
           @environment_source = NewRelic::Security::Agent::Configuration::EnvironmentSource.new
           @server_source = NewRelic::Security::Agent::Configuration::ServerSource.new
@@ -93,6 +116,14 @@ module NewRelic::Security
           @cache[:listen_port] = listen_port
         end
 
+        def traffic_start_time=(traffic_start_time)
+          @cache[:traffic_start_time] = traffic_start_time
+        end
+
+        def scan_start_time=(scan_start_time)
+          @cache[:scan_start_time] = scan_start_time
+        end
+
         def app_server=(app_server)
           @cache[:app_server] = app_server
         end
@@ -171,6 +202,19 @@ module NewRelic::Security
         def generate_key(entity_guid)
           ::OpenSSL::PKCS5.pbkdf2_hmac(entity_guid, entity_guid[0..15], 1024, 32, SHA1)
         end
+
+        def current_time_millis
+          (Time.now.to_f * 1000).to_i
+        end
+
+        def convert_to_regexp_list(value_list)
+          value_list.map do |value|
+            next unless value && !value.empty?
+            value = "^#{value}" if value[0] != '^'
+            value = "#{value}$" if value[-1] != '$'
+            /#{value}/
+          end
+        end
       end
     end
   end

data/lib/newrelic_security/agent/control/collector.rb

@@ -19,6 +19,8 @@ module NewRelic::Security
         def collect(case_type, args, event_category = nil, **keyword_args)
           return unless NewRelic::Security::Agent.config[:enabled]
           return if NewRelic::Security::Agent::Control::HTTPContext.get_context.nil? && NewRelic::Security::Agent::Control::GRPCContext.get_context.nil?
+          return if check_and_exclude_from_iast_scan_for_api
+          return if check_and_exclude_from_iast_scan_for_detection_category(case_type)
           args.map! { |file| Pathname.new(file).relative? ? File.join(Dir.pwd, file) : file } if [FILE_OPERATION, FILE_INTEGRITY].include?(case_type)
 
           event = NewRelic::Security::Agent::Control::Event.new(case_type, args, event_category)
@@ -44,7 +46,7 @@ module NewRelic::Security
           # Hence, considering only frame absolute_path & lineno for apiId calculation.
           user_frame_index = get_user_frame_index(stk)
           event.apiId = "#{case_type}-#{calculate_api_id(stk[0..user_frame_index].map { |frame| "#{frame.absolute_path}:#{frame.lineno}" }, event.httpRequest[:method], route)}"
-          stk.delete_if {|frame| frame.path.match(/newrelic_security/) || frame.path.match(/new_relic/)}
+          stk.delete_if { |frame| frame.path.match?(/newrelic_security/) || frame.path.match?(/new_relic/) }
           user_frame_index = get_user_frame_index(stk)
           return if case_type != REFLECTED_XSS && user_frame_index == -1 # TODO: Add log message here: "Filtered because User Stk frame NOT FOUND   \r\n"
           if user_frame_index != -1
@@ -59,8 +61,8 @@ module NewRelic::Security
           end
           event.stacktrace = stk[0..user_frame_index].map(&:to_s)
           NewRelic::Security::Agent.agent.event_processor.send_event(event)
-          if event.httpRequest[:headers].key?(NR_CSEC_FUZZ_REQUEST_ID) && event.apiId == event.httpRequest[:headers][NR_CSEC_FUZZ_REQUEST_ID].split(COLON_IAST_COLON)[0]
-            NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId] << event.id if NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId]
+          if event.httpRequest[:headers].key?(NR_CSEC_FUZZ_REQUEST_ID) && event.apiId == event.httpRequest[:headers][NR_CSEC_FUZZ_REQUEST_ID].split(COLON_IAST_COLON)[0] && NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId]
+            NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId] << event.id
           end
           event
         rescue Exception => exception
@@ -111,6 +113,35 @@ module NewRelic::Security
           }
         end
 
+        def check_and_exclude_from_iast_scan_for_api
+          NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.api'].each do |api|
+            return true if api&.match?(NewRelic::Security::Agent::Control::HTTPContext.get_context.url)
+          end
+          return false
+        end
+
+        def check_and_exclude_from_iast_scan_for_detection_category(case_type)
+          case case_type
+          when FILE_OPERATION, FILE_INTEGRITY
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access']
+          when SQL_DB_COMMAND
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection']
+          when NOSQL_DB_COMMAND
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection']
+          when LDAP
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection']
+          when SYSTEM_COMMAND
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.command_injection']
+          when XPATH
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection']
+          when HTTP_REQUEST
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ssrf']
+          when REFLECTED_XSS
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.rxss']
+          else
+            false
+          end
+        end
       end
     end
   end

data/lib/newrelic_security/agent/control/control_command.rb

@@ -49,8 +49,6 @@ module NewRelic::Security
               message_object[:arguments].each { |processed_id| NewRelic::Security::Agent.agent.iast_client.completed_requests.delete(processed_id) }
             when 100
               NewRelic::Security::Agent.logger.debug "Control command : '100', #{message_object.to_json}"
-              ::NewRelic::Agent.instance.events.notify(:security_policy_received, message_object[:data])
-              # TODO: Update policy from file here, if enabled.
             when 101
 
             when 102

data/lib/newrelic_security/agent/control/event.rb

@@ -30,7 +30,20 @@ module NewRelic::Security
           @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
           @httpRequest = Hash.new
           @httpResponse = Hash.new
-          @metaData = { :reflectedMetaData => { :listen_port => NewRelic::Security::Agent.config[:listen_port].to_s }, :appServerInfo => { :applicationDirectory => NewRelic::Security::Agent.config[:app_root], :serverBaseDirectory => NewRelic::Security::Agent.config[:app_root] } }
+          @metaData = {
+            :reflectedMetaData => {
+              :listen_port => NewRelic::Security::Agent.config[:listen_port].to_s
+            },
+            :appServerInfo => { 
+              :applicationDirectory => NewRelic::Security::Agent.config[:app_root], 
+              :serverBaseDirectory => NewRelic::Security::Agent.config[:app_root] 
+            },
+            :skipScanParameters => {
+              :header => NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.header'],
+              :query => NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.query'],
+              :body => NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.body']
+            }
+          }
           @linkingMetadata = add_linking_metadata
           @pid = pid
           @parameters = args

data/lib/newrelic_security/agent/control/event_processor.rb

@@ -48,6 +48,7 @@ module NewRelic::Security
           enqueue(event)
           if @first_event
             NewRelic::Security::Agent.init_logger.info "[STEP-8] => First event sent for validation. Security agent started successfully : #{event.to_json}"
+            NewRelic::Security::Agent.config.traffic_start_time = current_time_millis unless NewRelic::Security::Agent.config[:traffic_start_time]
             @first_event = false
           end
           event = nil
@@ -130,6 +131,10 @@ module NewRelic::Security
           NewRelic::Security::Agent.logger.error "Exception in health check thread, #{exception.inspect}"
         end
 
+        def current_time_millis
+          (Time.now.to_f * 1000).to_i
+        end
+
         def create_error_reporting_thread
           @error_reporting_thread = Thread.new {
             Thread.current.name = "newrelic_security_error_reporting_thread"

data/lib/newrelic_security/agent/control/event_subscriber.rb

@@ -7,19 +7,13 @@ module NewRelic::Security
               NewRelic::Security::Agent.logger.info "NewRelic server_source_configuration_added for pid : #{Process.pid}, Parent Pid : #{Process.ppid}"
               NewRelic::Security::Agent.init_logger.info "NewRelic server_source_configuration_added for pid : #{Process.pid}, Parent Pid : #{Process.ppid}"
               NewRelic::Security::Agent.config.update_server_config
-              if NewRelic::Security::Agent.config[:enabled] && !NewRelic::Security::Agent.config[:high_security]
-                NewRelic::Security::Agent.agent.init
+              if NewRelic::Security::Agent.config[:'security.enabled'] && !NewRelic::Security::Agent.config[:high_security]
+                Thread.new { NewRelic::Security::Agent.agent.scan_scheduler.init_via_scan_scheduler }
               else
                 NewRelic::Security::Agent.logger.info "New Relic Security is disabled by one of the user provided config `security.enabled` or `high_security`."
                 NewRelic::Security::Agent.init_logger.info "New Relic Security is disabled by one of the user provided config `security.enabled` or `high_security`."
               end
             }
-            ::NewRelic::Agent.instance.events.subscribe(:security_policy_received) { |received_policy| 
-              NewRelic::Security::Agent.logger.info "security_policy_received pid ::::::: #{Process.pid} #{Process.ppid}, #{received_policy}"
-              NewRelic::Security::Agent.config[:policy].merge!(received_policy)
-              NewRelic::Security::Agent.init_logger.info "[STEP-7] => Received and applied policy/configuration : #{received_policy}"
-              NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
-            }
         end
       end
     end

data/lib/newrelic_security/agent/control/health_check.rb

@@ -35,6 +35,9 @@ module NewRelic::Security
           @iastEventStats = {}
           @raspEventStats = {}
           @exitEventStats = {}
+          @procStartTime = NewRelic::Security::Agent.config[:process_start_time]
+          @trafficStartedTime = NewRelic::Security::Agent.config[:traffic_start_time]
+          @scanStartTime = NewRelic::Security::Agent.config[:scan_start_time]
         end
 
         def as_json

data/lib/newrelic_security/agent/control/http_context.rb

@@ -12,17 +12,20 @@ module NewRelic::Security
       REQUEST_METHOD = 'REQUEST_METHOD'
       HTTP_HOST = 'HTTP_HOST'
       PATH_INFO = 'PATH_INFO'
+      QUERY_STRING = 'QUERY_STRING'
       RACK_INPUT = 'rack.input'
       CGI_VARIABLES = ::Set.new(%W[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS HTTP_HOST PATH_INFO PATH_TRANSLATED REQUEST_URI QUERY_STRING REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER REQUEST_METHOD SCRIPT_NAME SERVER_NAME SERVER_PORT SERVER_PROTOCOL SERVER_SOFTWARE rack.url_scheme ])
+      REQUEST_BODY_LIMIT = 500 #KB
 
       class HTTPContext
         
-        attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter, :mutex
+        attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter, :mutex, :url
 
         def initialize(env)
           @time_stamp = current_time_millis
           @req = env.select { |key, _| CGI_VARIABLES.include? key}
           @method = @req[REQUEST_METHOD]
+          @url = "#{@req[PATH_INFO]}?#{@req[QUERY_STRING]}"
           @headers = env.select { |key, _| key.include?(HTTP_) }
           @headers = @headers.transform_keys{ |key| key[5..-1].gsub(UNDERSCORE, HYPHEN).downcase }
           request = Rack::Request.new(env) unless env.empty?
@@ -31,18 +34,18 @@ module NewRelic::Security
           strio = env[RACK_INPUT]
           if strio.instance_of?(::StringIO)
 						offset = strio.tell
-						@body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024) #after read, offset changes
+						@body = strio.read(REQUEST_BODY_LIMIT * 1024) #after read, offset changes
 						strio.seek(offset)
             # In case of Grape and Roda strio.read giving empty result, added below approach to handle such cases
             @body = strio.string if @body.nil? && strio.size > 0
           elsif defined?(::Rack) && defined?(::Rack::Lint::InputWrapper) && strio.instance_of?(::Rack::Lint::InputWrapper)
-						@body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024)
+						@body = strio.read(REQUEST_BODY_LIMIT * 1024)
           elsif defined?(::Protocol::Rack::Input) && defined?(::Protocol::Rack::Input) && strio.instance_of?(::Protocol::Rack::Input)
-            @body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024)
+            @body = strio.read(REQUEST_BODY_LIMIT * 1024)
           elsif defined?(::PhusionPassenger::Utils::TeeInput) && strio.instance_of?(::PhusionPassenger::Utils::TeeInput)
-						@body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024)
+						@body = strio.read(REQUEST_BODY_LIMIT * 1024)
           end
-          @data_truncated = @body && @body.size >= NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024
+          @data_truncated = @body && @body.size >= REQUEST_BODY_LIMIT * 1024
 					strio&.rewind
 					@body = @body.force_encoding(Encoding::UTF_8) if @body.is_a?(String)
           @cache = Hash.new

data/lib/newrelic_security/agent/control/iast_client.rb

@@ -17,9 +17,8 @@ module NewRelic::Security
       IS_GRPC = 'isGrpc'
       INPUT_CLASS = 'inputClass'
       SERVER_PORT_1 = 'serverPort'
-      PROBING = 'probing'
-      INTERVAL = 'interval'
       IS_GRPC_CLIENT_STREAM = 'isGrpcClientStream'
+      PROBING_INTERVAL = 5
 
       class IASTClient
         
@@ -54,6 +53,7 @@ module NewRelic::Security
               Thread.current.name = "newrelic_security_iast_thread-#{t}"
               loop do
                 fuzz_request = @fuzzQ.deq #thread blocks when the queue is empty
+                NewRelic::Security::Agent.config.scan_start_time = current_time_millis unless NewRelic::Security::Agent.config[:scan_start_time]
                 if fuzz_request.request[IS_GRPC]
                   fire_grpc_request(fuzz_request.id, fuzz_request.request, fuzz_request.reflected_metadata)
                 else
@@ -71,7 +71,8 @@ module NewRelic::Security
           @iast_data_transfer_request_processor_thread = Thread.new do
             Thread.current.name = "newrelic_security_iast_data_transfer_request_processor"
             loop do
-              sleep NewRelic::Security::Agent.config[:policy][VULNERABILITY_SCAN][IAST_SCAN][PROBING][INTERVAL]
+              # TODO: Check & remove this probing interval if not required, earlier this was used from policy sent by SE.
+              sleep PROBING_INTERVAL
               current_timestamp = current_time_millis
               cooldown_sleep_time = @cooldown_till_timestamp - current_timestamp
               sleep cooldown_sleep_time/1000 if cooldown_sleep_time > 0
@@ -84,9 +85,21 @@ module NewRelic::Security
               if batch_size > 100 && remaining_record_capacity > batch_size
                 iast_data_transfer_request = NewRelic::Security::Agent::Control::IASTDataTransferRequest.new
                 iast_data_transfer_request.batchSize = batch_size * 2
+                # TODO: Below calculation of batch_size overrides above logic and can be removed once below one is stablises or rate limit feature is released.
+                if NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit']
+                  batch_size =
+                    if NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] < 12
+                      1
+                    elsif NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] > 3600
+                      300
+                    else
+                      NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] / 12
+                    end
+                  iast_data_transfer_request.batchSize = batch_size
+                end
                 iast_data_transfer_request.pendingRequestIds = pending_request_ids.to_a
                 iast_data_transfer_request.completedRequests = completed_requests
-                NewRelic::Security::Agent.agent.event_processor.send_iast_data_transfer_request(iast_data_transfer_request)
+                NewRelic::Security::Agent.agent.event_processor&.send_iast_data_transfer_request(iast_data_transfer_request)
               end
             end
           end
@@ -122,18 +135,18 @@ module NewRelic::Security
         def fire_grpc_request(fuzz_request_id, request, reflected_metadata)
           service = Object.const_get(request[METHOD].split(SLASH)[0]).superclass
           method = request[METHOD].split(SLASH)[1]
-          @stub = service.rpc_stub_class.new("localhost:#{request[SERVER_PORT_1]}", :this_channel_is_insecure) unless @stub
+          @stub ||= service.rpc_stub_class.new("localhost:#{request[SERVER_PORT_1]}", :this_channel_is_insecure)
 
-          parsed_body =  request[BODY][1..-2].split(',')
-          if reflected_metadata[IS_GRPC_CLIENT_STREAM]
-            chunks_enum = Enumerator.new do |y|
+          parsed_body = request[BODY][1..-2].split(',')
+          chunks_enum = if reflected_metadata[IS_GRPC_CLIENT_STREAM]
+            Enumerator.new do |y|
               parsed_body.each do |b|
                 y << Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(b)
               end
             end
-          else
-            chunks_enum = Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(request[BODY])
-          end
+                        else
+            Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(request[BODY])
+                        end
           response = @stub.public_send(method, chunks_enum, metadata: request[HEADERS])
           # response = @stub.send(method, JSON.parse(request['body'], object_class: OpenStruct))
           # request[HEADERS].delete(VERSION) if request[HEADERS].key?(VERSION)

data/lib/newrelic_security/agent/control/scan_scheduler.rb

@@ -0,0 +1,77 @@
+require 'newrelic_security/parse-cron/cron_parser'
+
+module NewRelic::Security
+  module Agent
+    module Control
+      class ScanScheduler
+        def init_via_scan_scheduler
+          if NewRelic::Security::Agent.config[:'security.scan_schedule.delay'].positive?
+            NewRelic::Security::Agent.logger.info "IAST delay is set to: #{NewRelic::Security::Agent.config[:'security.scan_schedule.delay']}, current time: #{Time.now}"
+            start_agent_with_delay(NewRelic::Security::Agent.config[:'security.scan_schedule.delay']*60)
+          elsif !NewRelic::Security::Agent.config[:'security.scan_schedule.schedule'].to_s.empty?
+            cron_expression_task(NewRelic::Security::Agent.config[:'security.scan_schedule.schedule'], NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
+          else
+            NewRelic::Security::Agent.agent.init
+            NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
+            shutdown_at_duration_reached(NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
+          end
+        rescue StandardError => exception
+          NewRelic::Security::Agent.logger.error "Exception in IAST scan scheduler: #{exception.inspect} #{exception.backtrace}"
+          ::NewRelic::Agent.notice_error(exception)
+        end
+
+        def start_agent_with_delay(delay)
+          NewRelic::Security::Agent.logger.info "Security Agent delay scan time is set to: #{(delay/60).ceil} minutes when always_sample_traces is #{NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']}, current time: #{Time.now}"
+          if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
+            NewRelic::Security::Agent.agent.init unless NewRelic::Security::Agent.config[:enabled]
+            sleep delay if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
+          else
+            sleep delay
+            NewRelic::Security::Agent.agent.init
+          end
+          NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
+          shutdown_at_duration_reached(NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
+        end
+
+        def shutdown_at_duration_reached(duration)
+          shutdown_at = Time.now.to_i + duration
+          shut_down_time = (Time.now + duration).strftime("%a %d %b %Y %H:%M:%S")
+          return if duration <= 0
+          NewRelic::Security::Agent.logger.info "IAST Duration is set to: #{duration/60} minutes, timestamp: #{shut_down_time} time, current time: #{Time.now}"
+          @shutdown_monitor_thread = Thread.new do
+            Thread.current.name = "newrelic_security_shutdown_monitor_thread"
+            loop do
+              sleep 1
+              next if Time.now.to_i < shutdown_at
+              if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
+                NewRelic::Security::Agent.logger.info "Shutdown IAST Data transfer request processor only as 'security.scan_schedule.always_sample_traces' is #{NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']} now at current time: #{Time.now}"
+                NewRelic::Security::Agent.agent.iast_client&.iast_data_transfer_request_processor_thread&.kill
+              else
+                NewRelic::Security::Agent.logger.info "Shutdown IAST agent now at current time: #{Time.now}"
+                ::NewRelic::Agent.notice_error(StandardError.new("WS Connection closed by local"))
+                NewRelic::Security::Agent.agent.shutdown_security_agent
+              end
+              break
+            end
+          end
+        end
+
+        def cron_expression_task(schedule, duration)
+          @cron_parser = NewRelic::Security::ParseCron::CronParser.new(schedule)
+          loop do
+            next_run = @cron_parser.next(Time.now)
+            NewRelic::Security::Agent.logger.info "Next init via cron exp: #{schedule},  is scheduled at : #{next_run}"
+            delay = next_run - Time.now
+            if NewRelic::Security::Agent.agent.iast_client&.iast_data_transfer_request_processor_thread&.alive?
+              sleep delay > 2 ? delay - 2 : delay
+            else
+              start_agent_with_delay(delay) 
+            end
+            return if duration <= 0
+          end
+        end
+        
+      end
+    end
+  end
+end