newrelic_rpm 5.1.0.344 → 5.2.0.345

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 542ac1ce1a083c278c83f047f78ab8b3c8810a0916c87f83894a1f923d0a3fd2
-  data.tar.gz: 4e1eb213e1a4dac5877552f15e971bdf1b5005d2fb208a5364a8e1e59b9d25a0
+  metadata.gz: d6f27f55d62839f2426dcc139af19e97f815c70783944d4c67363a7d54a5e208
+  data.tar.gz: 6543b4fa585ab70291df98c295818ccb74eb4d2ab00a9ac3f5093c7b89656d4d
 SHA512:
-  metadata.gz: eaf4583a69257f2e22aaae8a6148a5b38d130f83e9585fd69b70ddaf6696e18d7da0a814e7d0046fa6fc6d1afd23f591f961194b56c3da4d86c97cc9d975da4e
-  data.tar.gz: 49de5e3b35b731ef61835a2f3fed0a82fcf7ff9af97db695c829cd3936b19bd7ecf1087ff69c5a755a743f203a1efec8c070d4587737e0419e364116bd827121
+  metadata.gz: 658044904957344ad87805e072168e9f366453733f0b9e42051230c2999be7170bfe5cac64c402e0ced3483e1f3d5ce091622c01d2ecb685e4354b41e6a11a38
+  data.tar.gz: ec9e153d940103faddd85fca6b00191e9214770aa5c7b5d1318fff41e3ff6d69f9720d2300847571ed1ba92b7d9c468fce5afc8dc6cfa711807405384d6cc94e

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # New Relic Ruby Agent Release Notes #
 
+  ## v5.2.0 ##
+
+  * Use priority sampling for errors and custom events
+
+    Priority sampling replaces the older reservoir event sampling method.
+    With this change, the agent will maintain randomness across a given
+    time period while improving coordination among transactions, errors,
+    and custom events.
+
+  * Bugfix for wrapping datastore operations
+
+    The agent will now complete the process of wrapping datastore
+    operations even if an error occurs during execution of a callback.
+
   ## v5.1.0 ##
 
   * Rails 5.2 support

data/lib/new_relic/agent/agent.rb

@@ -8,7 +8,6 @@ require 'net/http'
 require 'logger'
 require 'zlib'
 require 'stringio'
-require 'new_relic/agent/sampled_buffer'
 require 'new_relic/agent/autostart'
 require 'new_relic/agent/harvester'
 require 'new_relic/agent/hostname'
@@ -21,7 +20,6 @@ require 'new_relic/agent/event_listener'
 require 'new_relic/agent/cross_app_monitor'
 require 'new_relic/agent/distributed_trace_monitor'
 require 'new_relic/agent/synthetics_monitor'
-require 'new_relic/agent/synthetics_event_buffer'
 require 'new_relic/agent/transaction_event_recorder'
 require 'new_relic/agent/custom_event_aggregator'
 require 'new_relic/agent/sampler_collection'
@@ -183,7 +181,8 @@ module NewRelic
           return if !needs_restart ||
             !Agent.config[:agent_enabled] ||
             !Agent.config[:monitor_mode] ||
-            disconnected?
+            disconnected? ||
+            !control.security_settings_valid?
 
           ::NewRelic::Agent.logger.debug "Starting the worker thread in #{Process.pid} (parent #{Process.ppid}) after forking."
 
@@ -844,6 +843,20 @@ module NewRelic
 
             @service.agent_id = config_data['agent_run_id']
 
+            security_policies = config_data.delete('security_policies')
+
+            add_server_side_config(config_data)
+            add_security_policy_config(security_policies) if security_policies
+
+            log_connection!(config_data)
+            @transaction_rules = RulesEngine.create_transaction_rules(config_data)
+            @stats_engine.metric_rules = RulesEngine.create_metric_rules(config_data)
+
+            # If you're adding something else here to respond to the server-side config,
+            # use Agent.instance.events.subscribe(:finished_configuring) callback instead!
+          end
+
+          def add_server_side_config(config_data)
             if config_data['agent_config']
               ::NewRelic::Agent.logger.debug "Using config from server"
             end
@@ -851,13 +864,14 @@ module NewRelic
             ::NewRelic::Agent.logger.debug "Server provided config: #{config_data.inspect}"
             server_config = NewRelic::Agent::Configuration::ServerSource.new(config_data, Agent.config)
             Agent.config.replace_or_add_config(server_config)
-            log_connection!(config_data)
-
-            @transaction_rules = RulesEngine.create_transaction_rules(config_data)
-            @stats_engine.metric_rules = RulesEngine.create_metric_rules(config_data)
+          end
 
-            # If you're adding something else here to respond to the server-side config,
-            # use Agent.instance.events.subscribe(:finished_configuring) callback instead!
+          def add_security_policy_config(security_policies)
+            ::NewRelic::Agent.logger.info 'Installing security policies'
+            security_policy_source = NewRelic::Agent::Configuration::SecurityPolicySource.new(security_policies)
+            Agent.config.replace_or_add_config(security_policy_source)
+            # drop data collected before applying security policies
+            drop_buffered_data
           end
 
           class WaitOnConnectTimeout < StandardError

data/lib/new_relic/agent/configuration/default_source.rb

@@ -313,6 +313,13 @@ module NewRelic
           :allowed_from_server => false,
           :description => 'If <code>true</code>, enables <a href="https://docs.newrelic.com/docs/accounts-partnerships/accounts/security/high-security">high security mode</a>. Ensure you understand the implications of high security mode before enabling this setting.'
         },
+        :security_policies_token => {
+          :default => '',
+          :public => true,
+          :type => String,
+          :allowed_from_server => false,
+          :description => 'Applies Language Agent Security Policy settings.'
+        },
         :proxy_host => {
           :default => nil,
           :allow_nil => true,

data/lib/new_relic/agent/configuration/manager.rb

@@ -9,6 +9,7 @@ require 'new_relic/agent/configuration/default_source'
 require 'new_relic/agent/configuration/server_source'
 require 'new_relic/agent/configuration/environment_source'
 require 'new_relic/agent/configuration/high_security_source'
+require 'new_relic/agent/configuration/security_policy_source'
 
 module NewRelic
   module Agent
@@ -44,12 +45,13 @@ module NewRelic
 
         def remove_config_type(sym)
           source = case sym
-          when :high_security then @high_security_source
-          when :environment   then @environment_source
-          when :server        then @server_source
-          when :manual        then @manual_source
-          when :yaml          then @yaml_source
-          when :default       then @default_source
+          when :security_policy then @security_policy_source
+          when :high_security   then @high_security_source
+          when :environment     then @environment_source
+          when :server          then @server_source
+          when :manual          then @manual_source
+          when :yaml            then @yaml_source
+          when :default         then @default_source
           end
 
           remove_config(source)
@@ -57,12 +59,13 @@ module NewRelic
 
         def remove_config(source)
           case source
-          when HighSecuritySource then @high_security_source = nil
-          when EnvironmentSource  then @environment_source   = nil
-          when ServerSource       then @server_source        = nil
-          when ManualSource       then @manual_source        = nil
-          when YamlSource         then @yaml_source          = nil
-          when DefaultSource      then @default_source       = nil
+          when SecurityPolicySource then @security_policy_source = nil
+          when HighSecuritySource   then @high_security_source   = nil
+          when EnvironmentSource    then @environment_source     = nil
+          when ServerSource         then @server_source          = nil
+          when ManualSource         then @manual_source          = nil
+          when YamlSource           then @yaml_source            = nil
+          when DefaultSource        then @default_source         = nil
           else
             @configs_for_testing.delete_if {|src,lvl| src == source}
           end
@@ -78,12 +81,13 @@ module NewRelic
 
           invoke_callbacks(:add, source)
           case source
-          when HighSecuritySource then @high_security_source = source
-          when EnvironmentSource  then @environment_source   = source
-          when ServerSource       then @server_source        = source
-          when ManualSource       then @manual_source        = source
-          when YamlSource         then @yaml_source          = source
-          when DefaultSource      then @default_source       = source
+          when SecurityPolicySource then @security_policy_source = source
+          when HighSecuritySource   then @high_security_source   = source
+          when EnvironmentSource    then @environment_source     = source
+          when ServerSource         then @server_source          = source
+          when ManualSource         then @manual_source          = source
+          when YamlSource           then @yaml_source            = source
+          when DefaultSource        then @default_source         = source
           else
             NewRelic::Agent.logger.warn("Invalid config format; config will be ignored: #{source}")
           end
@@ -323,14 +327,15 @@ module NewRelic
 
         # Generally only useful during initial construction and tests
         def reset_to_defaults
-          @high_security_source = nil
-          @environment_source   = EnvironmentSource.new
-          @server_source        = nil
-          @manual_source        = nil
-          @yaml_source          = nil
-          @default_source       = DefaultSource.new
+          @security_policy_source = nil
+          @high_security_source   = nil
+          @environment_source     = EnvironmentSource.new
+          @server_source          = nil
+          @manual_source          = nil
+          @yaml_source            = nil
+          @default_source         = DefaultSource.new
 
-          @configs_for_testing  = []
+          @configs_for_testing    = []
 
           reset_cache
         end
@@ -350,13 +355,14 @@ module NewRelic
         end
 
         def delete_all_configs_for_testing
-          @high_security_source = nil
-          @environment_source   = nil
-          @server_source        = nil
-          @manual_source        = nil
-          @yaml_source          = nil
-          @default_source       = nil
-          @configs_for_testing  = []
+          @security_policy_source = nil
+          @high_security_source   = nil
+          @environment_source     = nil
+          @server_source          = nil
+          @manual_source          = nil
+          @yaml_source            = nil
+          @default_source         = nil
+          @configs_for_testing    = []
         end
 
         def num_configs_for_testing
@@ -370,7 +376,8 @@ module NewRelic
         private
 
         def config_stack
-          stack = [@high_security_source,
+          stack = [@security_policy_source,
+                   @high_security_source,
                    @environment_source,
                    @server_source,
                    @manual_source,

data/lib/new_relic/agent/configuration/security_policy_source.rb

@@ -0,0 +1,224 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/configuration/dotted_hash'
+
+module NewRelic
+  module Agent
+    module Configuration
+      class SecurityPolicySource < DottedHash
+        class << self
+          def enabled?(option)
+            Agent.config[option]
+          end
+
+          def record_sql_enabled?(option)
+            Agent.config[option] == 'obfuscated' ||
+              Agent.config[option] == 'raw' ||
+              false
+          end
+
+          def not_empty?(option)
+            !Agent.config[option].empty?
+          end
+
+          def change_setting(policies, option, new_value)
+            current_value = Agent.config[option]
+            unless current_value == new_value
+              NewRelic::Agent.logger.info \
+                "Setting changed: {#{option}: from #{current_value} " \
+                "to #{new_value}}. Source: SecurityPolicySource"
+            end
+            policies[option] = new_value
+          end
+        end
+
+        # The keys of the security settings map are the names of security
+        # policies received from the server. They map to multiple configuration
+        # options in the local config. There is a hash of metadata that
+        # corresponds to each configuration option with the following keys:
+        #
+        # option: the configuration option name
+        # supported: true if the agent has one or more corresponding
+        #   configuration options
+        # enabled_fn: a callable that takes the configuration option and returns
+        #   true if the option is enabled, false otherwise
+        # disabled_value: the value of the configuration option when it is
+        #   disabled
+        # permitted_fn: a callable, that will be executed if an option is
+        #   permitted by the security policy and is also enabled by the config
+
+        SECURITY_SETTINGS_MAP = {
+          "record_sql" => [
+            {
+              option:         :'transaction_tracer.record_sql',
+              supported:      true,
+              enabled_fn:     method(:record_sql_enabled?),
+              disabled_value: 'off',
+              permitted_fn:   proc { |policies|
+                change_setting(policies, :'transaction_tracer.record_sql', 'obfuscated')
+              }
+            },
+            {
+              option:         :'slow_sql.record_sql',
+              supported:      true,
+              enabled_fn:     method(:record_sql_enabled?),
+              disabled_value: 'off',
+              permitted_fn:   proc { |policies|
+                change_setting(policies, :'slow_sql.record_sql', 'obfuscated')
+              }
+            },
+            {
+              option:         :'mongo.capture_queries',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   proc{ |policies|
+                change_setting(policies, :'mongo.obfuscate_queries', true)
+              }
+            },
+            {
+              option:         :'transaction_tracer.record_redis_arguments',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            }
+          ],
+          "attributes_include" => [
+            {
+              option:         :'attributes.include',
+              supported:      true,
+              enabled_fn:     method(:not_empty?),
+              disabled_value: [],
+              permitted_fn:   nil
+            },
+            {
+              option:         :'transaction_tracer.attributes.include',
+              supported:      true,
+              enabled_fn:     method(:not_empty?),
+              disabled_value: [],
+              permitted_fn:   nil
+            },
+            {
+              option:         :'transaction_events.attributes.include',
+              supported:      true,
+              enabled_fn:     method(:not_empty?),
+              disabled_value: [],
+              permitted_fn:   nil
+            },
+            {
+              option:         :'error_collector.attributes.include',
+              supported:      true,
+              enabled_fn:     method(:not_empty?),
+              disabled_value: [],
+              permitted_fn:   nil
+            },
+            {
+              option:         :'browser_monitoring.attributes.include',
+              supported:      true,
+              enabled_fn:     method(:not_empty?),
+              disabled_value: [],
+              permitted_fn:   nil
+            }
+          ],
+          "allow_raw_exception_messages" => [
+            {
+              option:         :'strip_exception_messages.enabled',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            }
+          ],
+          "custom_events" => [
+            {
+              option:         :'custom_insights_events.enabled',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            }
+          ],
+          "custom_parameters" => [
+            {
+              option:         :'custom_attributes.enabled',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            }
+          ],
+          "custom_instrumentation_editor" => [
+            {
+              option:         nil,
+              supported:      false,
+              enabled_fn:     nil,
+              disabled_value: nil,
+              permitted_fn:   nil
+            }
+          ],
+          "message_parameters" => [
+            {
+              option:         :'message_tracer.segment_parameters.enabled',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            }
+          ],
+          "job_arguments" => [
+            {
+              option:         :'resque.capture_params',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            },
+            {
+              option:         :'sidekiq.capture_params',
+              supported:      true,
+              enabled_fn:     method(:enabled?),
+              disabled_value: false,
+              permitted_fn:   nil
+            }
+          ]
+        }
+
+        def initialize(security_policies)
+          super(build_overrides(security_policies))
+        end
+
+        ENABLED = "enabled".freeze
+        COLON_COLON = "::".freeze
+
+        def build_overrides(security_policies)
+          security_policies.inject({}) do |settings, (policy_name, policy_settings)|
+            SECURITY_SETTINGS_MAP[policy_name].each do |policy|
+              next unless policy[:supported]
+              if policy_settings[ENABLED]
+                if policy[:enabled_fn].call(policy[:option])
+                  if permitted_fn = policy[:permitted_fn]
+                    permitted_fn.call(settings)
+                  end
+                else
+                  config_source = Agent.config.source(policy[:option]).class.name.split(COLON_COLON).last
+                  NewRelic::Agent.logger.info \
+                    "Setting applied: {#{policy[:option]}: #{policy[:disabled_value]}}. " \
+                    "Source: #{config_source}"
+                end
+              else
+                settings[policy[:option]] =  policy[:disabled_value]
+                NewRelic::Agent.logger.info \
+                  "Setting applied: {#{policy[:option]}: #{policy[:disabled_value]}}. " \
+                  "Source: SecurityPolicySource"
+              end
+            end
+            settings
+          end
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/custom_event_aggregator.rb

@@ -12,6 +12,7 @@ module NewRelic
 
       TYPE             = 'type'.freeze
       TIMESTAMP        = 'timestamp'.freeze
+      PRIORITY         = 'priority'.freeze
       EVENT_TYPE_REGEX = /^[a-zA-Z0-9:_ ]+$/.freeze
 
       named :CustomEventAggregator
@@ -23,25 +24,35 @@ module NewRelic
           raise ArgumentError, "Expected Hash but got #{attributes.class}"
         end
 
+        return unless enabled?
+
         type = @type_strings[type]
         unless type =~ EVENT_TYPE_REGEX
           note_dropped_event(type)
           return false
         end
 
-        event = [
-          { TYPE => type, TIMESTAMP => Time.now.to_i },
-          AttributeProcessing.flatten_and_coerce(attributes)
-        ]
+        priority = attributes[:priority] || rand
 
         stored = @lock.synchronize do
-          @buffer.append(event)
+          @buffer.append(priority: priority) do
+            create_event(type, priority, attributes)
+          end
         end
         stored
       end
 
       private
 
+      def create_event(type, priority, attributes)
+        [
+          { TYPE => type, TIMESTAMP => Time.now.to_i,
+            PRIORITY => priority
+          },
+          AttributeProcessing.flatten_and_coerce(attributes)
+        ]
+      end
+
       def after_initialize
         @type_strings = Hash.new { |hash, key| hash[key] = key.to_s.freeze }
       end

data/lib/new_relic/agent/datastores.rb

@@ -121,11 +121,14 @@ module NewRelic
         begin
           result = yield
         ensure
-          if callback
-            elapsed_time = (Time.now - segment.start_time).to_f
-            callback.call(result, segment.name, elapsed_time)
+          begin
+            if callback
+                elapsed_time = (Time.now - segment.start_time).to_f
+                callback.call(result, segment.name, elapsed_time)
+            end
+          ensure
+            segment.finish if segment
           end
-          segment.finish if segment
         end
       end
 

data/lib/new_relic/agent/error_collector.rb

@@ -209,7 +209,7 @@ module NewRelic
         noticed_error = create_noticed_error(exception, options)
         error_trace_aggregator.add_to_error_queue(noticed_error)
         payload = state.current_transaction ? state.current_transaction.payload : nil
-        error_event_aggregator.append_event(noticed_error, payload)
+        error_event_aggregator.record(noticed_error, payload)
         exception
       rescue => e
         ::NewRelic::Agent.logger.warn("Failure when capturing error '#{exception}':", e)

data/lib/new_relic/agent/error_event_aggregator.rb

@@ -5,6 +5,7 @@
 
 require 'new_relic/agent/event_aggregator'
 require 'new_relic/agent/transaction_error_primitive'
+require 'new_relic/agent/priority_sampled_buffer'
 
 module NewRelic
   module Agent
@@ -13,12 +14,15 @@ module NewRelic
       named :ErrorEventAggregator
       capacity_key :'error_collector.max_event_samples_stored'
       enabled_key :'error_collector.capture_events'
+      buffer_class PrioritySampledBuffer
 
-      def append_event noticed_error, transaction_payload = nil
+      def record noticed_error, transaction_payload = nil
         return unless enabled?
 
+        priority = (transaction_payload && transaction_payload[:priority]) || rand
+
         @lock.synchronize do
-          @buffer.append do
+          @buffer.append(priority: priority) do
             create_event(noticed_error, transaction_payload)
           end
           notify_if_full

data/lib/new_relic/agent/event_aggregator.rb

@@ -2,7 +2,7 @@
 # This file is distributed under New Relic's license terms.
 # See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
 
-require 'new_relic/agent/sampled_buffer'
+require 'new_relic/agent/priority_sampled_buffer'
 
 module NewRelic
   module Agent
@@ -24,7 +24,7 @@ module NewRelic
           if klass
             @buffer_class = klass
           else
-            @buffer_class ||= SampledBuffer
+            @buffer_class ||= PrioritySampledBuffer
           end
         end
       end
@@ -79,7 +79,7 @@ module NewRelic
             @buffer.decrement_lifetime_counts_by samples.count
           end
 
-          samples.each { |s| @buffer.append s }
+          samples.each { |s| @buffer.append event: s }
         end
       end
 

data/lib/new_relic/agent/new_relic_service.rb

@@ -8,6 +8,7 @@ require 'new_relic/agent/audit_logger'
 require 'new_relic/agent/new_relic_service/encoders'
 require 'new_relic/agent/new_relic_service/marshaller'
 require 'new_relic/agent/new_relic_service/json_marshaller'
+require 'new_relic/agent/new_relic_service/security_policy_settings'
 
 module NewRelic
   module Agent
@@ -15,7 +16,7 @@ module NewRelic
       # Specifies the version of the agent's communication protocol with
       # the NewRelic hosted site.
 
-      PROTOCOL_VERSION = 15
+      PROTOCOL_VERSION = 16
 
       # 1f147a42: v10 (tag 3.5.3.17)
       # cf0d1ff1: v9 (tag 3.5.0)
@@ -68,16 +69,35 @@ module NewRelic
       end
 
       def connect(settings={})
-        if host = preconnect
-          @collector = NewRelic::Control.instance.server_from_host(host)
+        security_policies = nil
+        if response = preconnect
+          if host = response['redirect_host']
+            @collector = NewRelic::Control.instance.server_from_host(host)
+          end
+          if policies = response['security_policies']
+            security_policies = SecurityPolicySettings.preliminary_settings(policies)
+            settings.merge!(security_policies)
+          end
         end
         response = invoke_remote(:connect, [settings])
         self.agent_id = response['agent_run_id']
+        response.merge!(security_policies) if security_policies
         response
       end
 
       def preconnect
-        invoke_remote(:preconnect)
+        token = Agent.config[:security_policies_token]
+
+        if token && !token.empty?
+          response = invoke_remote(:preconnect, [{'security_policies_token' => token}])
+
+          validator = SecurityPolicySettings::Validator.new(response)
+          validator.validate_matching_agent_config!
+
+          response
+        else
+          invoke_remote(:preconnect, [])
+        end
       end
 
       def shutdown(time)

data/lib/new_relic/agent/new_relic_service/security_policy_settings.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+module NewRelic
+  module Agent
+    class NewRelicService
+      module SecurityPolicySettings
+        EXPECTED_SECURITY_POLICIES = %w(
+          record_sql
+          attributes_include
+          allow_raw_exception_messages
+          custom_events
+          custom_parameters
+          custom_instrumentation_editor
+          message_parameters
+          job_arguments).map(&:freeze)
+
+        def self.preliminary_settings(security_policies)
+          enabled_key = 'enabled'.freeze
+          settings = EXPECTED_SECURITY_POLICIES.inject({}) do |memo, policy_name|
+            memo[policy_name] =  {enabled_key => security_policies[policy_name][enabled_key]}
+            memo
+          end
+          {'security_policies' => settings}
+        end
+
+        class Validator
+          def initialize(preconnect_response)
+            @preconnect_policies = preconnect_response['security_policies'] || {}
+          end
+
+          def validate_matching_agent_config!
+            agent_keys = EXPECTED_SECURITY_POLICIES
+            all_server_keys = @preconnect_policies.keys
+            required = 'required'
+            required_server_keys = @preconnect_policies.keys.select do |key|
+              key if @preconnect_policies[key][required]
+            end
+
+            missing_from_agent = required_server_keys - agent_keys
+            unless missing_from_agent.empty?
+              message = "The agent received one or more required security policies \
+that it does not recognize and will shut down: #{missing_from_agent.join(',')}. \
+Please check if a newer agent version supports these policies or contact support."
+              raise NewRelic::Agent::UnrecoverableAgentException.new(message)
+            end
+
+            missing_from_server = agent_keys - all_server_keys
+            unless missing_from_server.empty?
+              message = "The agent did not receive one or more security policies \
+that it expected and will shut down: #{missing_from_server.join(',')}. Please \
+contact support."
+              raise NewRelic::Agent::UnrecoverableAgentException.new(message)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/priority_sampled_buffer.rb

@@ -3,10 +3,11 @@
 # See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
 
 require 'new_relic/agent/heap'
+require 'new_relic/agent/event_buffer'
 
 module NewRelic
   module Agent
-    class PrioritySampledBuffer < SampledBuffer
+    class PrioritySampledBuffer < EventBuffer
       PRIORITY_KEY = "priority".freeze
 
       attr_reader :seen_lifetime, :captured_lifetime
@@ -28,12 +29,17 @@ module NewRelic
         if full?
           priority ||= priority_for(event)
           if priority_for(@items[0]) < priority
-            @items[0] = event || blk.call
+            incoming = event || blk.call
+            @items[0] = incoming
             @items.fix(0)
+            incoming
+          else
+            nil
           end
         else
           @items << (event || blk.call)
           @captured_lifetime += 1
+          @items[-1]
         end
       end
 
@@ -52,6 +58,22 @@ module NewRelic
         @items.to_a.dup
       end
 
+      def decrement_lifetime_counts_by n
+        @captured_lifetime -= n
+        @seen_lifetime -= n
+      end
+
+      def sample_rate_lifetime
+        @captured_lifetime > 0 ? (@captured_lifetime.to_f / @seen_lifetime) : 0.0
+      end
+
+      def metadata
+        super.merge!(
+          :captured_lifetime => @captured_lifetime,
+          :seen_lifetime => @seen_lifetime
+        )
+      end
+
       private
 
       def increment_seen

data/lib/new_relic/agent/synthetics_event_aggregator.rb

@@ -4,30 +4,23 @@
 # See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
 
 require 'new_relic/agent/event_aggregator'
-require 'new_relic/agent/synthetics_event_buffer'
+require 'new_relic/agent/timestamp_sampled_buffer'
 
 module NewRelic
   module Agent
     class SyntheticsEventAggregator < EventAggregator
+      TIMESTAMP = 'timestamp'.freeze
 
       named :SyntheticsEventAggregator
       capacity_key :'synthetics.events_limit'
       enabled_key :'analytics_events.enabled'
-      buffer_class SyntheticsEventBuffer
+      buffer_class TimestampSampledBuffer
 
-      def append_or_reject event
+      def record event
         return unless enabled?
 
         @lock.synchronize do
-          @buffer.append_with_reject event
-        end
-      end
-
-      # slightly different semantics than the EventAggregator for merge
-      def merge! payload
-        _, events = payload
-        @lock.synchronize do
-          events.each { |e| @buffer.append_with_reject e}
+          @buffer.append event: event, priority: -event[0][TIMESTAMP]
         end
       end
 

data/lib/new_relic/agent/timestamp_sampled_buffer.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/heap'
+
+module NewRelic
+  module Agent
+    class TimestampSampledBuffer < PrioritySampledBuffer
+      TIMESTAMP_KEY = "timestamp".freeze
+
+      private
+
+      def priority_for(event)
+        -event[0][TIMESTAMP_KEY]
+      end
+    end
+  end
+end

data/lib/new_relic/agent/transaction_error_primitive.rb

@@ -30,6 +30,7 @@ module NewRelic
       SYNTHETICS_RESOURCE_ID_KEY     = "nr.syntheticsResourceId".freeze
       SYNTHETICS_JOB_ID_KEY          = "nr.syntheticsJobId".freeze
       SYNTHETICS_MONITOR_ID_KEY      = "nr.syntheticsMonitorId".freeze
+      PRIORITY_KEY                   = "priority".freeze
 
       def create noticed_error, payload
         [
@@ -54,6 +55,7 @@ module NewRelic
           attrs[NAME_KEY] = payload[:name]
           attrs[DURATION_KEY] = payload[:duration]
           attrs[SAMPLED_KEY] = payload[:'sampled'] if Agent.config[:'distributed_tracing.enabled']
+          attrs[PRIORITY_KEY] = payload[:'priority']
           append_synthetics payload, attrs
           append_cat payload, attrs
           append_distributed_trace_intrinsics payload, attrs

data/lib/new_relic/agent/transaction_event_aggregator.rb

@@ -17,7 +17,7 @@ module NewRelic
       enabled_key :'analytics_events.enabled'
       buffer_class PrioritySampledBuffer
 
-      def append priority: nil, event:nil, &blk
+      def record priority: nil, event:nil, &blk
         unless(event || priority && blk)
           raise ArgumentError, "Expected priority and block, or event"
         end
@@ -30,18 +30,6 @@ module NewRelic
         end
       end
 
-      def merge! payload, adjust_count = true
-        @lock.synchronize do
-          _, samples = payload
-
-          if adjust_count
-            @buffer.decrement_lifetime_counts_by samples.count
-          end
-
-          samples.each { |s| @buffer.append event: s }
-        end
-      end
-
       private
 
       def after_harvest metadata

data/lib/new_relic/agent/transaction_event_recorder.rb

@@ -24,10 +24,10 @@ module NewRelic
 
         if synthetics_event? payload
           event = create_event payload
-          _, rejected = synthetics_event_aggregator.append_or_reject event
-          transaction_event_aggregator.append event: event if rejected
+          result = synthetics_event_aggregator.record event
+          transaction_event_aggregator.record event: event if result.nil?
         else
-          transaction_event_aggregator.append(priority: payload[:priority]) { create_event(payload) }
+          transaction_event_aggregator.record(priority: payload[:priority]) { create_event(payload) }
         end
       end
 

data/lib/new_relic/control/instance_methods.rb

@@ -68,7 +68,10 @@ module NewRelic
         Module.send :include, NewRelic::Agent::MethodTracer
         init_config(options)
         NewRelic::Agent.agent = NewRelic::Agent::Agent.instance
-        if Agent.config[:agent_enabled] && !NewRelic::Agent.instance.started?
+
+        if !security_settings_valid?
+          handle_invalid_security_settings
+        elsif Agent.config[:agent_enabled] && !NewRelic::Agent.instance.started?
           start_agent
           install_instrumentation
         elsif !Agent.config[:agent_enabled]
@@ -100,12 +103,26 @@ module NewRelic
         config_file_path = @config_file_override || Agent.config[:config_path]
         Agent.config.replace_or_add_config(Agent::Configuration::YamlSource.new(config_file_path, env))
 
-        if Agent.config[:high_security]
+        if security_settings_valid? && Agent.config[:high_security]
           Agent.logger.info("Installing high security configuration based on local configuration")
           Agent.config.replace_or_add_config(Agent::Configuration::HighSecuritySource.new(Agent.config))
         end
       end
 
+      def security_settings_valid?
+        !Agent.config[:high_security] ||
+          Agent.config[:security_policies_token].empty?
+      end
+
+      def handle_invalid_security_settings
+        NewRelic::Agent.logger.error "Security Policies and High Security " \
+          "Mode cannot both be present in the agent configuration. If " \
+          "Security Policies have been set for your account, please " \
+          "ensure the security_policies_token is set but high_security is " \
+          "disabled (default)."
+        install_shim
+      end
+
       # Install the real agent into the Agent module, and issue the start command.
       def start_agent
         @started_in_env = self.env

data/lib/new_relic/version.rb

@@ -11,7 +11,7 @@ module NewRelic
     end
 
     MAJOR = 5
-    MINOR = 1
+    MINOR = 2
     TINY  = 0
 
     begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: newrelic_rpm
 version: !ruby/object:Gem::Version
-  version: 5.1.0.344
+  version: 5.2.0.345
 platform: ruby
 authors:
 - Matthew Wear
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-25 00:00:00.000000000 Z
+date: 2018-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -217,6 +217,7 @@ files:
 - lib/new_relic/agent/configuration/manager.rb
 - lib/new_relic/agent/configuration/manual_source.rb
 - lib/new_relic/agent/configuration/mask_defaults.rb
+- lib/new_relic/agent/configuration/security_policy_source.rb
 - lib/new_relic/agent/configuration/server_source.rb
 - lib/new_relic/agent/configuration/yaml_source.rb
 - lib/new_relic/agent/cross_app_monitor.rb
@@ -331,6 +332,7 @@ files:
 - lib/new_relic/agent/new_relic_service/encoders.rb
 - lib/new_relic/agent/new_relic_service/json_marshaller.rb
 - lib/new_relic/agent/new_relic_service/marshaller.rb
+- lib/new_relic/agent/new_relic_service/security_policy_settings.rb
 - lib/new_relic/agent/null_logger.rb
 - lib/new_relic/agent/obfuscator.rb
 - lib/new_relic/agent/parameter_filtering.rb
@@ -343,7 +345,6 @@ files:
 - lib/new_relic/agent/rules_engine.rb
 - lib/new_relic/agent/rules_engine/replacement_rule.rb
 - lib/new_relic/agent/rules_engine/segment_terms_rule.rb
-- lib/new_relic/agent/sampled_buffer.rb
 - lib/new_relic/agent/sampler.rb
 - lib/new_relic/agent/sampler_collection.rb
 - lib/new_relic/agent/samplers/cpu_sampler.rb
@@ -351,7 +352,6 @@ files:
 - lib/new_relic/agent/samplers/memory_sampler.rb
 - lib/new_relic/agent/samplers/object_sampler.rb
 - lib/new_relic/agent/samplers/vm_sampler.rb
-- lib/new_relic/agent/sized_buffer.rb
 - lib/new_relic/agent/sql_sampler.rb
 - lib/new_relic/agent/stats.rb
 - lib/new_relic/agent/stats_engine.rb
@@ -359,13 +359,13 @@ files:
 - lib/new_relic/agent/stats_engine/stats_hash.rb
 - lib/new_relic/agent/supported_versions.rb
 - lib/new_relic/agent/synthetics_event_aggregator.rb
-- lib/new_relic/agent/synthetics_event_buffer.rb
 - lib/new_relic/agent/synthetics_monitor.rb
 - lib/new_relic/agent/system_info.rb
 - lib/new_relic/agent/threading/agent_thread.rb
 - lib/new_relic/agent/threading/backtrace_node.rb
 - lib/new_relic/agent/threading/backtrace_service.rb
 - lib/new_relic/agent/threading/thread_profile.rb
+- lib/new_relic/agent/timestamp_sampled_buffer.rb
 - lib/new_relic/agent/transaction.rb
 - lib/new_relic/agent/transaction/abstract_segment.rb
 - lib/new_relic/agent/transaction/attributes.rb
@@ -484,7 +484,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: New Relic Ruby Agent

data/lib/new_relic/agent/sampled_buffer.rb

@@ -1,68 +0,0 @@
-# encoding: utf-8
-# This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
-
-# This class acts like an Array with a fixed capacity that randomly samples
-# from a stream of items such that the probability of each item being included
-# in the Array is equal. It uses reservoir sampling in order to achieve this:
-# http://xlinux.nist.gov/dads/HTML/reservoirSampling.html
-
-require 'new_relic/agent/event_buffer'
-
-module NewRelic
-  module Agent
-    class SampledBuffer < EventBuffer
-      attr_reader :seen_lifetime, :captured_lifetime
-
-      def initialize(capacity)
-        super
-        @captured_lifetime = 0
-        @seen_lifetime     = 0
-      end
-
-      def append(x = nil, &blk)
-        @seen += 1
-        @seen_lifetime += 1
-        append_event(x, &blk)
-      end
-
-      def append_event(x = nil, &blk)
-        raise ArgumentError, "Expected argument or block, but received both" if x && blk
-
-        if @items.size < @capacity
-          x = blk.call if block_given?
-          @items << x
-          @captured_lifetime += 1
-          return x
-        else
-          m = rand(@seen) # [0, @seen)
-          if m < @capacity
-            x = blk.call if block_given?
-            @items[m] = x
-            return x
-          else
-            # discard current sample
-            return nil
-          end
-        end
-      end
-
-      def decrement_lifetime_counts_by n
-        @captured_lifetime -= n
-        @seen_lifetime -= n
-      end
-
-      def sample_rate_lifetime
-        @captured_lifetime > 0 ? (@captured_lifetime.to_f / @seen_lifetime) : 0.0
-      end
-
-      def metadata
-        super.merge!(
-          :captured_lifetime => @captured_lifetime,
-          :seen_lifetime => @seen_lifetime
-        )
-      end
-    end
-  end
-end
-

data/lib/new_relic/agent/sized_buffer.rb

@@ -1,23 +0,0 @@
-# encoding: utf-8
-# This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
-
-require 'new_relic/agent/event_buffer'
-
-module NewRelic
-  module Agent
-    class SizedBuffer < EventBuffer
-
-      def append_event(x)
-        if @items.size < @capacity
-          @items << x
-          return x
-        else
-          return nil
-        end
-      end
-
-    end
-  end
-end
-

data/lib/new_relic/agent/synthetics_event_buffer.rb

@@ -1,40 +0,0 @@
-# encoding: utf-8
-# This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
-
-require 'new_relic/agent/sized_buffer'
-
-module NewRelic
-  module Agent
-    class SyntheticsEventBuffer < SizedBuffer
-
-      def append_with_reject(x)
-        @seen += 1
-        if full?
-          timestamp = timestamp_for(x)
-          latest_event = @items.max_by do |item|
-            timestamp_for(item)
-          end
-
-          if timestamp < timestamp_for(latest_event)
-            # Make room!
-            @items.delete(latest_event)
-            return [append_event(x), latest_event]
-          else
-            return [nil, x]
-          end
-        else
-          return [append_event(x), nil]
-        end
-      end
-
-      TIMESTAMP = "timestamp".freeze
-
-      def timestamp_for(event)
-        main_event, _ = event
-        main_event[TIMESTAMP]
-      end
-    end
-  end
-end
-