rollbar 2.8.3 → 3.6.0

data/lib/rollbar/scrubbers/url.rb

@@ -6,34 +6,66 @@ require 'rollbar/language_support'
 module Rollbar
   module Scrubbers
     class URL
-      attr_reader :regex
-      attr_reader :scrub_user
-      attr_reader :scrub_password
-      attr_reader :randomize_scrub_length
+      SCRUB_ALL = :scrub_all
 
-      def initialize(options = {})
-        @regex = build_regex(options[:scrub_fields])
-        @scrub_user = options[:scrub_user]
-        @scrub_password = options[:scrub_password]
-        @randomize_scrub_length = options.fetch(:randomize_scrub_length, true)
+      def self.call(*args)
+        new.call(*args)
       end
 
-      def call(url)
-        return url unless Rollbar::LanguageSupport.can_scrub_url?
+      def call(options = {})
+        url = ascii_encode(options[:url])
+
+        filter(url,
+               build_regex(options[:scrub_fields]),
+               options[:scrub_user],
+               options[:scrub_password],
+               options.fetch(:randomize_scrub_length, true),
+               options[:scrub_fields].include?(SCRUB_ALL),
+               build_whitelist_regex(options[:whitelist] || []))
+      rescue StandardError => e
+        message = '[Rollbar] There was an error scrubbing the url: ' \
+          "#{e}, options: #{options.inspect}"
+        Rollbar.logger.error(message)
+        url
+      end
+
+      private
+
+      def ascii_encode(url)
+        # In some cases non-ascii characters won't be properly encoded, so we do it here.
+        #
+        # The standard encoders (the CGI and URI methods) are not reliable when
+        # the query string is already embedded in the full URL, but the inconsistencies
+        # are limited to issues with characters in the ascii range. (For example,
+        # the '#' if it appears in an unexpected place.) For escaping non-ascii,
+        # they are all OK, so we'll take care to skip the ascii chars.
+
+        return url if url.ascii_only?
+
+        # Iterate each char and only escape non-ascii characters.
+        url.each_char.map { |c| c.ascii_only? ? c : CGI.escape(c) }.join
+      end
+
+      def build_whitelist_regex(whitelist)
+        fields = whitelist.find_all { |f| f.is_a?(String) || f.is_a?(Symbol) }
+        return unless fields.any?
+
+        Regexp.new(fields.map { |val| /\A#{Regexp.escape(val.to_s)}\z/ }.join('|'))
+      end
 
+      def filter(url, regex, scrub_user, scrub_password, randomize_scrub_length,
+                 scrub_all, whitelist)
         uri = URI.parse(url)
 
-        uri.user = filter_user(uri.user)
-        uri.password = filter_password(uri.password)
-        uri.query = filter_query(uri.query)
+        uri.user = filter_user(uri.user, scrub_user, randomize_scrub_length)
+        uri.password = filter_password(uri.password, scrub_password,
+                                       randomize_scrub_length)
+        uri.query = filter_query(uri.query, regex, randomize_scrub_length, scrub_all,
+                                 whitelist)
 
         uri.to_s
-      rescue
-        url
       end
 
-      private
-
       # Builds a regex to match with any of the received fields.
       # The built regex will also match array params like 'user_ids[]'.
       def build_regex(fields)
@@ -42,23 +74,26 @@ module Rollbar
         Regexp.new("^#{fields_or}$")
       end
 
-      def filter_user(user)
-        scrub_user && user ? filtered_value(user) : user
+      def filter_user(user, scrub_user, randomize_scrub_length)
+        scrub_user && user ? filtered_value(user, randomize_scrub_length) : user
       end
 
-      def filter_password(password)
-        scrub_password && password ? filtered_value(password) : password
+      def filter_password(password, scrub_password, randomize_scrub_length)
+        if scrub_password && password
+          filtered_value(password,
+                         randomize_scrub_length)
+        else
+          password
+        end
       end
 
-      def filter_query(query)
+      def filter_query(query, regex, randomize_scrub_length, scrub_all, whitelist)
         return query unless query
 
         params = decode_www_form(query)
 
-        encoded_query = encode_www_form(filter_query_params(params))
-
-        # We want this to rebuild array params like foo[]=1&foo[]=2
-        CGI.unescape(encoded_query)
+        encode_www_form(filter_query_params(params, regex, randomize_scrub_length,
+                                            scrub_all, whitelist))
       end
 
       def decode_www_form(query)
@@ -66,29 +101,49 @@ module Rollbar
       end
 
       def encode_www_form(params)
-        URI.encode_www_form(params)
+        restore_square_brackets(URI.encode_www_form(params))
+      end
+
+      def restore_square_brackets(query)
+        # We want this to rebuild array params like foo[]=1&foo[]=2
+        #
+        # URI.encode_www_form follows the spec at
+        # https://url.spec.whatwg.org/#concept-urlencoded-serializer
+        # and percent encodes square brackets. Here we change them back.
+        query.gsub('%5B', '[').gsub('%5D', ']')
       end
 
-      def filter_query_params(params)
+      def filter_query_params(params, regex, randomize_scrub_length, scrub_all,
+                              whitelist)
         params.map do |key, value|
-          [key, filter_key?(key) ? filtered_value(value) : value]
+          [key,
+           if filter_key?(key, regex, scrub_all,
+                          whitelist)
+             filtered_value(value, randomize_scrub_length)
+           else
+             value
+           end]
         end
       end
 
-      def filter_key?(key)
-        !!(key =~ regex)
+      def filter_key?(key, regex, scrub_all, whitelist)
+        !(whitelist === key) && (scrub_all || regex === key)
       end
 
-      def filtered_value(value)
+      def filtered_value(value, randomize_scrub_length)
         if randomize_scrub_length
           random_filtered_value
         else
-          '*' * (value.length rescue 8)
+          '*' * (begin
+            value.length
+          rescue StandardError
+            8
+          end)
         end
       end
 
       def random_filtered_value
-        '*' * (rand(5) + 3)
+        '*' * rand(3..7)
       end
     end
   end

data/lib/rollbar/scrubbers.rb

@@ -1,4 +1,17 @@
 module Rollbar
   module Scrubbers
+    module_function
+
+    def scrub_value(_value)
+      if Rollbar.configuration.randomize_scrub_length
+        random_filtered_value
+      else
+        '*' * 6
+      end
+    end
+
+    def random_filtered_value
+      '*' * rand(3..7)
+    end
   end
 end

data/lib/rollbar/truncation/frames_strategy.rb

@@ -1,4 +1,5 @@
 require 'rollbar/truncation/mixin'
+require 'rollbar/util'
 
 module Rollbar
   module Truncation
@@ -10,7 +11,7 @@ module Rollbar
       end
 
       def call(payload)
-        new_payload = payload.clone
+        new_payload = payload
         body = new_payload['data']['body']
 
         if body['trace_chain']

data/lib/rollbar/truncation/min_body_strategy.rb

@@ -1,4 +1,5 @@
 require 'rollbar/truncation/mixin'
+require 'rollbar/util'
 
 module Rollbar
   module Truncation
@@ -10,8 +11,7 @@ module Rollbar
       end
 
       def call(payload)
-        new_payload = payload.clone
-        body = new_payload['data']['body']
+        body = payload['data']['body']
 
         if body['trace_chain']
           body['trace_chain'] = body['trace_chain'].map do |trace_data|
@@ -21,8 +21,7 @@ module Rollbar
           body['trace'] = truncate_trace_data(body['trace'])
         end
 
-
-        dump(new_payload)
+        dump(payload)
       end
 
       def truncate_trace_data(trace_data)

data/lib/rollbar/truncation/mixin.rb

@@ -9,7 +9,7 @@ module Rollbar
         result.bytesize > MAX_PAYLOAD_SIZE
       end
 
-      def select_frames(frames, range = 150)
+      def select_frames(frames, range = 50)
         return frames unless frames.count > range * 2
 
         frames[0, range] + frames[-range, range]

data/lib/rollbar/truncation/remove_any_key_strategy.rb

@@ -0,0 +1,126 @@
+require 'rollbar/util'
+
+module Rollbar
+  module Truncation
+    class RemoveAnyKeyStrategy
+      include ::Rollbar::Truncation::Mixin
+
+      attr_accessor :payload, :data, :sizes, :extracted_title
+
+      def self.call(payload)
+        new(payload).call
+      end
+
+      def initialize(payload)
+        @payload = payload
+        @data = payload['data']
+        @extracted_title = extract_title(data['body']) if data['body']
+      end
+
+      def call
+        remove_unknown_root_keys
+
+        json_payload = remove_oversized_data_keys
+
+        return json_payload if json_payload
+
+        dump(payload)
+      end
+
+      def remove_unknown_root_keys
+        payload.keys.reject { |key| root_keys.include?(key) }.each do |key|
+          truncation_key['root'] ||= {}
+          size = dump(payload.delete(key)).bytesize
+          truncation_key['root'][key] = "unknown root key removed, size: #{size} bytes"
+        end
+      end
+
+      def remove_oversized_data_keys
+        data_keys.keys.sort { |a, b| data_keys[b] <=> data_keys[a] }.each do |key|
+          json_payload = remove_key_and_return_payload(key)
+
+          return json_payload unless truncate?(json_payload)
+        end
+
+        false
+      end
+
+      def remove_key_and_return_payload(key)
+        size = data_keys[key]
+
+        data.delete(key)
+
+        replace_message_body if key == 'body'
+
+        truncation_key[key] = "key removed, size: #{size} bytes"
+
+        dump(payload)
+      end
+
+      def replace_message_body
+        data['body'] = message_key
+        data['title'] ||= extracted_title if extracted_title
+      end
+
+      def truncation_key
+        @truncation_key ||=
+          # initialize the diagnostic key for truncation
+          (data['notifier']['diagnostic'] ||= {}) &&
+          (data['notifier']['diagnostic']['truncation'] ||= {})
+      end
+
+      def root_keys
+        # Valid keys in root of payload
+        %w[access_token data]
+      end
+
+      def skip_keys
+        # Don't try to truncate these data keys
+        %w[notifier uuid title platform language framework level]
+      end
+
+      def message_key
+        # use this message if data.body gets removed
+        {
+          'message' => {
+            'body' => 'Payload keys removed due to oversized payload. See diagnostic key'
+          }
+        }
+      end
+
+      def extract_title(body)
+        return body['message']['body'] if body['message'] && body['message']['body']
+        return extract_title_from_trace(body['trace']) if body['trace']
+
+        return unless body['trace_chain'] && body['trace_chain'][0]
+
+        extract_title_from_trace(body['trace_chain'][0])
+      end
+
+      def extract_title_from_trace(trace)
+        exception = trace['exception']
+
+        "#{exception['class']}: #{exception['message']}"
+      end
+
+      def data_keys
+        @data_keys ||= {}.tap do |hash|
+          data.keys.reject { |key| skip_keys.include?(key) }.each do |key|
+            set_key_size(key, hash)
+          end
+        end
+      end
+
+      def set_key_size(key, hash)
+        size = dump(data[key]).bytesize
+        hash[key] = size
+      rescue ::JSON::GeneratorError
+        hash[key] = 0 # don't try to truncate non JSON object
+
+        # Log it
+        truncation_key['non_json_keys'] ||= {}
+        truncation_key['non_json_keys'][key] = data[key].class
+      end
+    end
+  end
+end

data/lib/rollbar/truncation/remove_extra_strategy.rb

@@ -0,0 +1,37 @@
+require 'rollbar/util'
+
+module Rollbar
+  module Truncation
+    class RemoveExtraStrategy
+      include ::Rollbar::Truncation::Mixin
+
+      def self.call(payload)
+        new.call(payload)
+      end
+
+      def call(payload)
+        body = payload['data']['body']
+
+        delete_message_extra(body)
+        delete_trace_chain_extra(body)
+        delete_trace_extra(body)
+
+        dump(payload)
+      end
+
+      def delete_message_extra(body)
+        body['message'].delete('extra') if body['message'] && body['message']['extra']
+      end
+
+      def delete_trace_chain_extra(body)
+        return unless body['trace_chain'] && body['trace_chain'][0]['extra']
+
+        body['trace_chain'][0].delete('extra')
+      end
+
+      def delete_trace_extra(body)
+        body['trace'].delete('extra') if body['trace'] && body['trace']['extra']
+      end
+    end
+  end
+end

data/lib/rollbar/truncation/remove_request_strategy.rb

@@ -0,0 +1,21 @@
+require 'rollbar/util'
+
+module Rollbar
+  module Truncation
+    class RemoveRequestStrategy
+      include ::Rollbar::Truncation::Mixin
+
+      def self.call(payload)
+        new.call(payload)
+      end
+
+      def call(payload)
+        data = payload['data']
+
+        data.delete('request') if data['request']
+
+        dump(payload)
+      end
+    end
+  end
+end

data/lib/rollbar/truncation/strings_strategy.rb

@@ -6,7 +6,7 @@ module Rollbar
     class StringsStrategy
       include ::Rollbar::Truncation::Mixin
 
-      STRING_THRESHOLDS = [1024, 512, 256]
+      STRING_THRESHOLDS = [1024, 512, 256, 128].freeze
 
       def self.call(payload)
         new.call(payload)
@@ -14,13 +14,12 @@ module Rollbar
 
       def call(payload)
         result = nil
-        new_payload = payload.clone
 
         STRING_THRESHOLDS.each do |threshold|
           truncate_proc = truncate_strings_proc(threshold)
 
-          ::Rollbar::Util.iterate_and_update(new_payload, truncate_proc)
-          result = dump(new_payload)
+          ::Rollbar::Util.iterate_and_update(payload, truncate_proc)
+          result = dump(payload)
 
           break unless truncate?(result)
         end
@@ -30,7 +29,9 @@ module Rollbar
 
       def truncate_strings_proc(threshold)
         proc do |value|
-          if value.is_a?(String) && value.bytesize > threshold
+          # Rollbar::Util.truncate will operate on characters, not bytes,
+          # so use value.length, not bytesize.
+          if value.is_a?(String) && value.length > threshold
             Rollbar::Util.truncate(value, threshold)
           else
             value

data/lib/rollbar/truncation.rb

@@ -4,23 +4,29 @@ require 'rollbar/truncation/raw_strategy'
 require 'rollbar/truncation/frames_strategy'
 require 'rollbar/truncation/strings_strategy'
 require 'rollbar/truncation/min_body_strategy'
+require 'rollbar/truncation/remove_request_strategy'
+require 'rollbar/truncation/remove_extra_strategy'
+require 'rollbar/truncation/remove_any_key_strategy'
 
 module Rollbar
   module Truncation
     extend ::Rollbar::Truncation::Mixin
 
-    MAX_PAYLOAD_SIZE = 128 * 1024 # 128kb
+    MAX_PAYLOAD_SIZE = 512 * 1024 # 512kb
     STRATEGIES = [RawStrategy,
                   FramesStrategy,
                   StringsStrategy,
-                  MinBodyStrategy
-                 ]
+                  MinBodyStrategy,
+                  RemoveRequestStrategy,
+                  RemoveExtraStrategy,
+                  RemoveAnyKeyStrategy].freeze
 
-    def self.truncate(payload)
+    def self.truncate(payload, attempts = [])
       result = nil
 
       STRATEGIES.each do |strategy|
         result = strategy.call(payload)
+        attempts << result.bytesize
         break unless truncate?(result)
       end
 

data/lib/rollbar/util/hash.rb

@@ -1,24 +1,55 @@
 module Rollbar
   module Util
-    module Hash
-      def self.deep_stringify_keys(hash)
+    module Hash # :nodoc:
+      def self.deep_stringify_keys(hash, seen = {})
+        seen.compare_by_identity
+        return if seen[hash]
+
+        seen[hash] = true
+        replace_seen_children(hash, seen)
+
         hash.reduce({}) do |h, (key, value)|
-          h[key.to_s] = map_value(value, :deep_stringify_keys)
+          h[key.to_s] = map_value(value, :deep_stringify_keys, seen)
 
           h
         end
       end
 
-      def self.map_value(thing, m)
+      def self.map_value(thing, meth, seen)
         case thing
         when ::Hash
-          send(m, thing)
+          send(meth, thing, seen)
         when Array
-          thing.map { |v| map_value(v, m) }
+          if seen[thing]
+            thing
+          else
+            seen[thing] = true
+            replace_seen_children(thing, seen)
+            thing.map { |v| map_value(v, meth, seen) }
+          end
         else
           thing
         end
       end
+
+      def self.replace_seen_children(thing, seen)
+        case thing
+        when ::Hash
+          thing.keys.each do |key| # rubocop:disable Style/HashEachMethods
+            if seen[thing[key]]
+              thing[key] =
+                "removed circular reference: #{thing[key]}"
+            end
+          end
+        when Array
+          thing.each_with_index do |_, i|
+            if seen[thing[i]]
+              thing[i] =
+                "removed circular reference: #{thing[i]}"
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/rollbar/util/ip_anonymizer.rb

@@ -0,0 +1,33 @@
+module Rollbar
+  module Util
+    module IPAnonymizer
+      require 'ipaddr'
+
+      def self.anonymize_ip(ip_string)
+        return ip_string unless Rollbar.configuration.anonymize_user_ip
+
+        ip = IPAddr.new(ip_string)
+        return anonymize_ipv6 ip if ip.ipv6?
+        return anonymize_ipv4 ip if ip.ipv4?
+      rescue StandardError
+        nil
+      end
+
+      def self.anonymize_ipv4(ip)
+        ip_parts = ip.to_s.split '.'
+
+        ip_parts[ip_parts.count - 1] = '0'
+
+        IPAddr.new(ip_parts.join('.')).to_s
+      end
+
+      def self.anonymize_ipv6(ip)
+        ip_parts = ip.to_s.split ':'
+
+        ip_string = "#{ip_parts[0..2].join(':')}:0000:0000:0000:0000:0000"
+
+        IPAddr.new(ip_string).to_s
+      end
+    end
+  end
+end

data/lib/rollbar/util/ip_obfuscator.rb

@@ -13,7 +13,7 @@ module Rollbar
         obfuscated_ip_int32 = ip_int32 ^ secret_int32 % (2 << 31)
 
         IPAddr.new(obfuscated_ip_int32, Socket::AF_INET).to_s
-      rescue
+      rescue StandardError
         nil
       end
     end