newrelic_rpm 3.12.0.288 → 9.10.2

data/lib/new_relic/agent/connect/response_handler.rb

@@ -0,0 +1,58 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module Connect
+      class ResponseHandler
+        def initialize(agent, config)
+          @agent = agent
+          @config = config
+        end
+
+        # Takes a hash of configuration data returned from the
+        # server and uses it to set local variables and to
+        # initialize various parts of the agent that are configured
+        # separately.
+        #
+        # Can accommodate most arbitrary data - anything extra is
+        # ignored unless we say to do something with it here.
+        def configure_agent(config_data)
+          return if config_data.nil?
+
+          @agent.agent_id = config_data['agent_run_id']
+
+          security_policies = config_data.delete('security_policies')
+
+          add_server_side_config(config_data)
+          add_security_policy_config(security_policies) if security_policies
+
+          @agent.transaction_rules = RulesEngine.create_transaction_rules(config_data)
+          @agent.stats_engine.metric_rules = RulesEngine.create_metric_rules(config_data)
+
+          # If you're adding something else here to respond to the server-side config,
+          # use Agent.instance.events.subscribe(:initial_configuration_complete) callback instead!
+        end
+
+        def add_server_side_config(config_data)
+          if config_data['agent_config']
+            ::NewRelic::Agent.logger.debug('Using config from server')
+          end
+
+          ::NewRelic::Agent.logger.debug("Server provided config: #{config_data.inspect}")
+          server_config = NewRelic::Agent::Configuration::ServerSource.new(config_data, @config)
+          @config.replace_or_add_config(server_config)
+        end
+
+        def add_security_policy_config(security_policies)
+          ::NewRelic::Agent.logger.info('Installing security policies')
+          security_policy_source = NewRelic::Agent::Configuration::SecurityPolicySource.new(security_policies)
+          @config.replace_or_add_config(security_policy_source)
+          # drop data collected before applying security policies
+          @agent.drop_buffered_data
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/custom_event_aggregator.rb

@@ -1,92 +1,105 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
-require 'new_relic/agent/sized_buffer'
+require 'new_relic/agent/event_aggregator'
 require 'new_relic/agent/attribute_processing'
 
 module NewRelic
   module Agent
-    class CustomEventAggregator
+    class CustomEventAggregator < EventAggregator
       include NewRelic::Coerce
 
-      TYPE             = 'type'.freeze
-      TIMESTAMP        = 'timestamp'.freeze
+      TYPE = 'type'.freeze
+      TIMESTAMP = 'timestamp'.freeze
+      PRIORITY = 'priority'.freeze
       EVENT_TYPE_REGEX = /^[a-zA-Z0-9:_ ]+$/.freeze
+      MAX_ATTRIBUTE_COUNT = 64
+      MAX_ATTRIBUTE_SIZE = 4095
+      MAX_NAME_SIZE = 255
 
-      DEFAULT_CAPACITY_KEY = :'custom_insights_events.max_samples_stored'
-
-      def initialize
-        @lock         = Mutex.new
-        @buffer       = SampledBuffer.new(NewRelic::Agent.config[DEFAULT_CAPACITY_KEY])
-        @type_strings = Hash.new { |hash, key| hash[key] = key.to_s.freeze }
-        register_config_callbacks
-      end
-
-      def register_config_callbacks
-        NewRelic::Agent.config.register_callback(DEFAULT_CAPACITY_KEY) do |max_samples|
-          NewRelic::Agent.logger.debug "CustomEventAggregator max_samples set to #{max_samples}"
-          @lock.synchronize do
-            @buffer.capacity = max_samples
-          end
-        end
-      end
+      named :CustomEventAggregator
+      capacity_key :'custom_insights_events.max_samples_stored'
+      enabled_key :'custom_insights_events.enabled'
 
       def record(type, attributes)
-        unless attributes.is_a? Hash
+        unless attributes.is_a?(Hash)
           raise ArgumentError, "Expected Hash but got #{attributes.class}"
         end
 
+        return unless enabled?
+
         type = @type_strings[type]
-        unless type =~ EVENT_TYPE_REGEX
+        unless EVENT_TYPE_REGEX.match?(type)
           note_dropped_event(type)
           return false
         end
 
-        event = [
-          { TYPE => type, TIMESTAMP => Time.now.to_i },
-          AttributeProcessing.flatten_and_coerce(attributes)
-        ]
+        priority = attributes[:priority] || rand
 
         stored = @lock.synchronize do
-          @buffer.append(event)
+          @buffer.append(priority: priority) do
+            create_event(type, priority, attributes)
+          end
         end
         stored
       end
 
-      def harvest!
-        results = []
-        drop_count = 0
-        @lock.synchronize do
-          results.concat(@buffer.to_a)
-          drop_count += @buffer.num_dropped
-          @buffer.reset!
-        end
-        note_dropped_events(results.size, drop_count)
-        results
+      private
+
+      def create_event(type, priority, attributes)
+        [
+          {TYPE => type,
+           TIMESTAMP => Process.clock_gettime(Process::CLOCK_REALTIME).to_i,
+           PRIORITY => priority},
+          create_custom_event_attributes(type, attributes)
+        ]
       end
 
-      def note_dropped_events(captured_count, dropped_count)
-        total_count = captured_count + dropped_count
-        if dropped_count > 0
-          NewRelic::Agent.logger.warn("Dropped #{dropped_count} custom events out of #{total_count}.")
+      def create_custom_event_attributes(type, attributes)
+        result = AttributeProcessing.flatten_and_coerce(attributes)
+
+        if result.size > MAX_ATTRIBUTE_COUNT
+          NewRelic::Agent.logger.warn("Custom event attributes are limited to #{MAX_ATTRIBUTE_COUNT}. Discarding #{result.size - MAX_ATTRIBUTE_COUNT} attributes")
+          result = result.first(MAX_ATTRIBUTE_COUNT)
         end
-        engine = NewRelic::Agent.instance.stats_engine
-        engine.tl_record_supportability_metric_count("Events/Customer/Seen"   ,    total_count)
-        engine.tl_record_supportability_metric_count("Events/Customer/Sent"   , captured_count)
-        engine.tl_record_supportability_metric_count("Events/Customer/Dropped",  dropped_count)
-      end
 
-      def merge!(events)
-        @lock.synchronize do
-          events.each do |event|
-            @buffer.append(event)
+        result.each_with_object({}) do |(key, val), new_result|
+          # name is limited to 255
+          if key.is_a?(String) && key.length > MAX_NAME_SIZE
+            key = key[0, MAX_NAME_SIZE]
+          end
+
+          # value is limited to 4095 except for LLM content-related events
+          if val.is_a?(String) && val.length > MAX_ATTRIBUTE_SIZE
+            val = val[0, MAX_ATTRIBUTE_SIZE] unless NewRelic::Agent::LLM.exempt_event_attribute?(type, key)
           end
+
+          new_result[key] = val
+        end
+      end
+
+      def after_initialize
+        @type_strings = Hash.new { |hash, key| hash[key] = key.to_s.freeze }
+      end
+
+      def after_harvest(metadata)
+        dropped_count = metadata[:seen] - metadata[:captured]
+        note_dropped_events(metadata[:seen], dropped_count)
+        record_supportability_metrics(metadata[:seen], metadata[:captured], dropped_count)
+      end
+
+      def note_dropped_events(total_count, dropped_count)
+        if dropped_count > 0
+          NewRelic::Agent.logger.warn("Dropped #{dropped_count} custom events out of #{total_count}.")
         end
       end
 
-      def reset!
-        @lock.synchronize { @buffer.reset! }
+      def record_supportability_metrics(total_count, captured_count, dropped_count)
+        engine = NewRelic::Agent.instance.stats_engine
+        engine.tl_record_supportability_metric_count('Events/Customer/Seen', total_count)
+        engine.tl_record_supportability_metric_count('Events/Customer/Sent', captured_count)
+        engine.tl_record_supportability_metric_count('Events/Customer/Dropped', dropped_count)
       end
 
       def note_dropped_event(type)
@@ -94,7 +107,6 @@ module NewRelic
           "Invalid event type name '#{type}', not recording.")
         @buffer.note_dropped
       end
-
     end
   end
 end

data/lib/new_relic/agent/database/explain_plan_helpers.rb

@@ -0,0 +1,138 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/database/obfuscator'
+require 'new_relic/agent/database/postgres_explain_obfuscator'
+
+module NewRelic
+  module Agent
+    module Database
+      module ExplainPlanHelpers
+        SUPPORTED_ADAPTERS_FOR_EXPLAIN = [:postgres, :mysql2, :mysql, :sqlite]
+        SELECT = 'select'.freeze
+
+        def is_select?(sql)
+          NewRelic::Agent::Database.parse_operation_from_query(sql) == SELECT
+        end
+
+        def parameterized?(sql)
+          Obfuscator.instance.obfuscate_single_quote_literals(sql) =~ /\$\d+/
+        end
+
+        # SQL containing a semicolon in the middle (with something
+        # other than whitespace after it) may contain two or more
+        # queries.  It's not safe to EXPLAIN this kind of expression,
+        # since it could lead to executing unwanted SQL.
+        #
+        MULTIPLE_QUERIES = Regexp.new(';\s*\S+')
+
+        def multiple_queries?(sql)
+          sql =~ MULTIPLE_QUERIES
+        end
+
+        def handle_exception_in_explain
+          yield
+        rescue => e
+          begin
+            # guarantees no throw from explain_sql
+            ::NewRelic::Agent.logger.error('Error getting query plan:', e)
+            nil
+          rescue
+            # double exception. throw up your hands
+            nil
+          end
+        end
+
+        def process_resultset(results, adapter)
+          if adapter == :postgres
+            return process_explain_results_postgres(results)
+          elsif defined?(::ActiveRecord::Result) && results.is_a?(::ActiveRecord::Result)
+            # Note if adapter is mysql, will only have headers, not values
+            return [results.columns, results.rows]
+          elsif results.is_a?(String)
+            return string_explain_plan_results(results)
+          end
+
+          case adapter
+          when :mysql2
+            process_explain_results_mysql2(results)
+          when :mysql
+            process_explain_results_mysql(results)
+          when :sqlite
+            process_explain_results_sqlite(results)
+          end
+        end
+
+        QUERY_PLAN = 'QUERY PLAN'.freeze
+
+        def process_explain_results_postgres(results)
+          if defined?(::ActiveRecord::Result) && results.is_a?(::ActiveRecord::Result)
+            query_plan_string = results.rows.join("\n")
+          elsif results.is_a?(String)
+            query_plan_string = results
+          else
+            lines = []
+            results.each { |row| lines << row[QUERY_PLAN] }
+            query_plan_string = lines.join("\n")
+          end
+
+          unless NewRelic::Agent::Database.record_sql_method == :raw
+            query_plan_string = NewRelic::Agent::Database::PostgresExplainObfuscator.obfuscate(query_plan_string)
+          end
+          values = query_plan_string.split("\n").map { |line| [line] }
+
+          [[QUERY_PLAN], values]
+        end
+
+        # Sequel returns explain plans as just one big pre-formatted String
+        # In that case, we send a nil headers array, and the single string
+        # wrapped in an array for the values.
+        # Note that we don't use this method for Postgres explain plans, since
+        # they need to be passed through the explain plan obfuscator first.
+        def string_explain_plan_results(results)
+          [nil, [results]]
+        end
+
+        def process_explain_results_mysql(results)
+          headers = []
+          values = []
+          if results.is_a?(Array)
+            # We're probably using the jdbc-mysql gem for JRuby, which will give
+            # us an array of hashes.
+            headers = results.first.keys
+            results.each do |row|
+              values << headers.map { |h| row[h] }
+            end
+          else
+            # We're probably using the native mysql driver gem, which will give us
+            # a Mysql::Result object that responds to each_hash
+            results.each_hash do |row|
+              headers = row.keys
+              values << headers.map { |h| row[h] }
+            end
+          end
+          [headers, values]
+        end
+
+        def process_explain_results_mysql2(results)
+          headers = results.fields
+          values = []
+          results.each { |row| values << row }
+          [headers, values]
+        end
+
+        SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
+
+        def process_explain_results_sqlite(results)
+          headers = SQLITE_EXPLAIN_COLUMNS
+          values = []
+          results.each do |row|
+            values << headers.map { |h| row[h] }
+          end
+          [headers, values]
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/database/obfuscation_helpers.rb

@@ -1,76 +1,100 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 module NewRelic
   module Agent
     module Database
       module ObfuscationHelpers
-        # Note that the following two regexes are applied to a reversed version
-        # of the query. This is why the backslash escape sequences (\' and \")
-        # appear reversed within them.
-        #
-        # Note that some database adapters (notably, PostgreSQL with
-        # standard_conforming_strings on and MySQL with NO_BACKSLASH_ESCAPES on)
-        # do not apply special treatment to backslashes within quoted string
-        # literals. We don't have an easy way of determining whether the
-        # database connection from which a query was captured was operating in
-        # one of these modes, but the obfuscation is done in such a way that it
-        # should not matter.
-        #
-        # Reversing the query string before obfuscation allows us to get around
-        # the fact that a \' appearing within a string may or may not terminate
-        # the string, because we know that a string cannot *start* with a \'.
-        REVERSE_SINGLE_QUOTES_REGEX = /'(?:''|'\\|[^'])*'/
-        REVERSE_ANY_QUOTES_REGEX    = /'(?:''|'\\|[^'])*'|"(?:""|"\\|[^"])*"/
+        COMPONENTS_REGEX_MAP = {
+          :single_quotes => /'(?:[^']|'')*?(?:\\'.*|'(?!'))/,
+          :double_quotes => /"(?:[^"]|"")*?(?:\\".*|"(?!"))/,
+          :dollar_quotes => /(\$(?!\d)[^$]*?\$).*?(?:\1|$)/,
+          :uuids => /\{?(?:[0-9a-fA-F]\-*){32}\}?/,
+          :numeric_literals => /-?\b(?:[0-9]+\.)?[0-9]+([eE][+-]?[0-9]+)?\b/,
+          :boolean_literals => /\b(?:true|false|null)\b/i,
+          :hexadecimal_literals => /0x[0-9a-fA-F]+/,
+          :comments => /(?:#|--).*?(?=\r|\n|$)/i,
+          :multi_line_comments => /\/\*(?:[^\/]|\/[^*])*?(?:\*\/|\/\*.*)/,
+          :oracle_quoted_strings => /q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)/
+        }
 
-        NUMERICS_REGEX = /\b\d+\b/
-
-        # We take a conservative, overly-aggressive approach to obfuscating
-        # comments, and drop everything from the query after encountering any
-        # character sequence that could be a comment initiator. We do this after
-        # removal of string literals to avoid accidentally over-obfuscating when
-        # a string literal contains a comment initiator.
-        SQL_COMMENT_REGEX = Regexp.new('(?:/\*|--|#).*', Regexp::MULTILINE).freeze
+        DIALECT_COMPONENTS = {
+          :fallback => COMPONENTS_REGEX_MAP.keys,
+          :mysql => [:single_quotes, :double_quotes, :numeric_literals, :boolean_literals,
+            :hexadecimal_literals, :comments, :multi_line_comments],
+          :postgres => [:single_quotes, :dollar_quotes, :uuids, :numeric_literals,
+            :boolean_literals, :comments, :multi_line_comments],
+          :sqlite => [:single_quotes, :numeric_literals, :boolean_literals, :hexadecimal_literals,
+            :comments, :multi_line_comments],
+          :oracle => [:single_quotes, :oracle_quoted_strings, :numeric_literals, :comments,
+            :multi_line_comments],
+          :cassandra => [:single_quotes, :uuids, :numeric_literals, :boolean_literals,
+            :hexadecimal_literals, :comments, :multi_line_comments]
+        }
 
         # We use these to check whether the query contains any quote characters
         # after obfuscation. If so, that's a good indication that the original
-        # query was malformed, and so our obfuscation can't reliabily find
+        # query was malformed, and so our obfuscation can't reliably find
         # literals. In such a case, we'll replace the entire query with a
         # placeholder.
-        LITERAL_SINGLE_QUOTE = "'".freeze
-        LITERAL_DOUBLE_QUOTE = '"'.freeze
+        CLEANUP_REGEX = {
+          :mysql => /'|"|\/\*|\*\//,
+          :mysql2 => /'|"|\/\*|\*\//,
+          :postgres => /'|\/\*|\*\/|\$(?!\?)/,
+          :sqlite => /'|\/\*|\*\//,
+          :cassandra => /'|\/\*|\*\//,
+          :oracle => /'|\/\*|\*\//,
+          :oracle_enhanced => /'|\/\*|\*\//
+        }
 
         PLACEHOLDER = '?'.freeze
+        FAILED_TO_OBFUSCATE_MESSAGE = 'Failed to obfuscate SQL query - quote characters remained after obfuscation'.freeze
 
         def obfuscate_single_quote_literals(sql)
-          obfuscated = sql.reverse
-          obfuscated.gsub!(REVERSE_SINGLE_QUOTES_REGEX, PLACEHOLDER)
-          obfuscated.reverse!
-          obfuscated
-        end
+          return sql unless sql&.match?(COMPONENTS_REGEX_MAP[:single_quotes])
 
-        def obfuscate_quoted_literals(sql)
-          obfuscated = sql.reverse
-          obfuscated.gsub!(REVERSE_ANY_QUOTES_REGEX, PLACEHOLDER)
-          obfuscated.reverse!
-          obfuscated
+          sql.gsub(COMPONENTS_REGEX_MAP[:single_quotes], PLACEHOLDER)
         end
 
-        def obfuscate_numeric_literals(sql)
-          sql.gsub(NUMERICS_REGEX, PLACEHOLDER)
+        def self.generate_regex(dialect)
+          components = DIALECT_COMPONENTS[dialect]
+          Regexp.union(components.map { |component| COMPONENTS_REGEX_MAP[component] })
         end
 
-        def remove_comments(sql)
-          sql.gsub(SQL_COMMENT_REGEX, PLACEHOLDER)
-        end
+        MYSQL_COMPONENTS_REGEX = self.generate_regex(:mysql)
+        POSTGRES_COMPONENTS_REGEX = self.generate_regex(:postgres)
+        SQLITE_COMPONENTS_REGEX = self.generate_regex(:sqlite)
+        ORACLE_COMPONENTS_REGEX = self.generate_regex(:oracle)
+        CASSANDRA_COMPONENTS_REGEX = self.generate_regex(:cassandra)
+        FALLBACK_REGEX = self.generate_regex(:fallback)
 
-        def contains_single_quotes?(str)
-          str.include?(LITERAL_SINGLE_QUOTE)
+        def obfuscate(sql, adapter)
+          case adapter
+          when :mysql, :mysql2
+            regex = MYSQL_COMPONENTS_REGEX
+          when :postgres
+            regex = POSTGRES_COMPONENTS_REGEX
+          when :sqlite
+            regex = SQLITE_COMPONENTS_REGEX
+          when :oracle, :oracle_enhanced
+            regex = ORACLE_COMPONENTS_REGEX
+          when :cassandra
+            regex = CASSANDRA_COMPONENTS_REGEX
+          else
+            regex = FALLBACK_REGEX
+          end
+          obfuscated = sql.gsub(regex, PLACEHOLDER)
+          obfuscated = FAILED_TO_OBFUSCATE_MESSAGE if detect_unmatched_pairs(obfuscated, adapter)
+          obfuscated
         end
 
-        def contains_quotes?(str)
-          str.include?(LITERAL_SINGLE_QUOTE) || str.include?(LITERAL_DOUBLE_QUOTE)
+        def detect_unmatched_pairs(obfuscated, adapter)
+          if CLEANUP_REGEX[adapter]
+            CLEANUP_REGEX[adapter].match(obfuscated)
+          else
+            CLEANUP_REGEX[:mysql].match(obfuscated)
+          end
         end
       end
     end

data/lib/new_relic/agent/database/obfuscator.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 require 'new_relic/agent/database/obfuscation_helpers'
 
@@ -13,8 +13,8 @@ module NewRelic
 
         attr_reader :obfuscator
 
-        QUERY_TOO_LARGE_MESSAGE     = "Query too large (over 16k characters) to safely obfuscate"
-        FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
+        QUERY_TOO_LARGE_MESSAGE = 'Query too large (over 16k characters) to safely obfuscate'.freeze
+        ELLIPSIS = '...'.freeze
 
         def initialize
           reset
@@ -48,31 +48,13 @@ module NewRelic
         end
 
         def default_sql_obfuscator(sql)
-          if sql[-3,3] == '...'
-            return QUERY_TOO_LARGE_MESSAGE
-          end
-
           stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
-          obfuscate_double_quotes = stmt.adapter.to_s !~ /postgres|sqlite/
-
-          obfuscated = obfuscate_numeric_literals(stmt)
 
-          if obfuscate_double_quotes
-            obfuscated = obfuscate_quoted_literals(obfuscated)
-            obfuscated = remove_comments(obfuscated)
-            if contains_quotes?(obfuscated)
-              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
-            end
-          else
-            obfuscated = obfuscate_single_quote_literals(obfuscated)
-            obfuscated = remove_comments(obfuscated)
-            if contains_single_quotes?(obfuscated)
-              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
-            end
+          if stmt.sql.end_with?(ELLIPSIS)
+            return QUERY_TOO_LARGE_MESSAGE
           end
 
-
-          obfuscated.to_s # return back to a regular String
+          obfuscate(stmt.sql, stmt.adapter).to_s
         end
       end
     end

data/lib/new_relic/agent/database/postgres_explain_obfuscator.rb

@@ -1,9 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
-
-require 'new_relic/agent/database'
-require 'new_relic/agent/database/obfuscation_helpers'
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 module NewRelic
   module Agent
@@ -13,9 +10,9 @@ module NewRelic
 
         # Note that this regex can't be shared with the ones in the
         # Database::Obfuscator class because here we don't look for
-        # backslash-escaped strings (and those regexes are backwards).
+        # backslash-escaped strings.
         QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
-        LABEL_LINE_REGEX     = /^([^:\n]*:\s+).*$/.freeze
+        LABEL_LINE_REGEX = /^([^:\n]*:\s+).*$/.freeze
 
         def obfuscate(explain)
           # First, we replace all single-quoted strings.
@@ -38,7 +35,7 @@ module NewRelic
           # All parts of the query that can appear in the explain output are
           # prefixed with "<label>: ", so we want to preserve the label, but
           # remove the rest of the line.
-          explain.gsub!(LABEL_LINE_REGEX,   '\1?')
+          explain.gsub!(LABEL_LINE_REGEX, '\1?')
           explain
         end
       end