activesupport 6.0.6.1 → 7.1.3.2

data/lib/active_support/messages/metadata.rb

@@ -1,80 +1,146 @@
 # frozen_string_literal: true
 
 require "time"
+require "active_support/json"
+require_relative "serializer_with_fallback"
 
 module ActiveSupport
-  module Messages #:nodoc:
-    class Metadata #:nodoc:
-      def initialize(message, expires_at = nil, purpose = nil)
-        @message, @purpose = message, purpose
-        @expires_at = expires_at.is_a?(String) ? parse_expires_at(expires_at) : expires_at
-      end
+  module Messages # :nodoc:
+    module Metadata # :nodoc:
+      singleton_class.attr_accessor :use_message_serializer_for_metadata
+
+      ENVELOPE_SERIALIZERS = [
+        *SerializerWithFallback::SERIALIZERS.values,
+        ActiveSupport::JSON,
+        ::JSON,
+        Marshal,
+      ]
+
+      TIMESTAMP_SERIALIZERS = [
+        SerializerWithFallback::SERIALIZERS.fetch(:message_pack),
+        SerializerWithFallback::SERIALIZERS.fetch(:message_pack_allow_marshal),
+      ]
 
-      def as_json(options = {})
-        { _rails: { message: @message, exp: @expires_at, pur: @purpose } }
+      ActiveSupport.on_load(:message_pack) do
+        ENVELOPE_SERIALIZERS << ActiveSupport::MessagePack
+        TIMESTAMP_SERIALIZERS << ActiveSupport::MessagePack
       end
 
-      class << self
-        def wrap(message, expires_at: nil, expires_in: nil, purpose: nil)
-          if expires_at || expires_in || purpose
-            JSON.encode new(encode(message), pick_expiry(expires_at, expires_in), purpose)
+      private
+        def serialize_with_metadata(data, **metadata)
+          has_metadata = metadata.any? { |k, v| v }
+
+          if has_metadata && !use_message_serializer_for_metadata?
+            data_string = serialize_to_json_safe_string(data)
+            envelope = wrap_in_metadata_legacy_envelope({ "message" => data_string }, **metadata)
+            serialize_to_json(envelope)
           else
-            message
+            data = wrap_in_metadata_envelope({ "data" => data }, **metadata) if has_metadata
+            serialize(data)
           end
         end
 
-        def verify(message, purpose)
-          extract_metadata(message).verify(purpose)
-        end
-
-        private
-          def pick_expiry(expires_at, expires_in)
-            if expires_at
-              expires_at.utc.iso8601(3)
-            elsif expires_in
-              Time.now.utc.advance(seconds: expires_in).iso8601(3)
+        def deserialize_with_metadata(message, **expected_metadata)
+          if dual_serialized_metadata_envelope_json?(message)
+            envelope = deserialize_from_json(message)
+            extracted = extract_from_metadata_envelope(envelope, **expected_metadata)
+            deserialize_from_json_safe_string(extracted["message"])
+          else
+            deserialized = deserialize(message)
+            if metadata_envelope?(deserialized)
+              extract_from_metadata_envelope(deserialized, **expected_metadata)["data"]
+            elsif expected_metadata.none? { |k, v| v }
+              deserialized
+            else
+              throw :invalid_message_content, "missing metadata"
             end
           end
+        end
 
-          def extract_metadata(message)
-            data = JSON.decode(message) rescue nil
+        def use_message_serializer_for_metadata?
+          Metadata.use_message_serializer_for_metadata && Metadata::ENVELOPE_SERIALIZERS.include?(serializer)
+        end
 
-            if data.is_a?(Hash) && data.key?("_rails")
-              new(decode(data["_rails"]["message"]), data["_rails"]["exp"], data["_rails"]["pur"])
-            else
-              new(message)
-            end
-          end
+        def wrap_in_metadata_envelope(hash, expires_at: nil, expires_in: nil, purpose: nil)
+          expiry = pick_expiry(expires_at, expires_in)
+          hash["exp"] = expiry if expiry
+          hash["pur"] = purpose.to_s if purpose
+          { "_rails" => hash }
+        end
+
+        def wrap_in_metadata_legacy_envelope(hash, expires_at: nil, expires_in: nil, purpose: nil)
+          expiry = pick_expiry(expires_at, expires_in)
+          hash["exp"] = expiry
+          hash["pur"] = purpose
+          { "_rails" => hash }
+        end
+
+        def extract_from_metadata_envelope(envelope, purpose: nil)
+          hash = envelope["_rails"]
 
-          def encode(message)
-            ::Base64.strict_encode64(message)
+          if hash["exp"] && Time.now.utc >= parse_expiry(hash["exp"])
+            throw :invalid_message_content, "expired"
           end
 
-          def decode(message)
-            ::Base64.strict_decode64(message)
+          if hash["pur"].to_s != purpose.to_s
+            throw :invalid_message_content, "mismatched purpose"
           end
-      end
 
-      def verify(purpose)
-        @message if match?(purpose) && fresh?
-      end
+          hash
+        end
 
-      private
-        def match?(purpose)
-          @purpose.to_s == purpose.to_s
+        def metadata_envelope?(object)
+          object.is_a?(Hash) && object.key?("_rails")
         end
 
-        def fresh?
-          @expires_at.nil? || Time.now.utc < @expires_at
+        def dual_serialized_metadata_envelope_json?(string)
+          string.start_with?('{"_rails":{"message":')
+        end
+
+        def pick_expiry(expires_at, expires_in)
+          expiry = if expires_at
+            expires_at.utc
+          elsif expires_in
+            Time.now.utc.advance(seconds: expires_in)
+          end
+
+          unless Metadata::TIMESTAMP_SERIALIZERS.include?(serializer)
+            expiry = expiry&.iso8601(3)
+          end
+
+          expiry
         end
 
-        def parse_expires_at(expires_at)
-          if ActiveSupport.use_standard_json_time_format
+        def parse_expiry(expires_at)
+          if !expires_at.is_a?(String)
+            expires_at
+          elsif ActiveSupport.use_standard_json_time_format
             Time.iso8601(expires_at)
           else
             Time.parse(expires_at)
           end
         end
+
+        def serialize_to_json(data)
+          ActiveSupport::JSON.encode(data)
+        end
+
+        def deserialize_from_json(serialized)
+          ActiveSupport::JSON.decode(serialized)
+        rescue ::JSON::ParserError => error
+          # Throw :invalid_message_format instead of :invalid_message_serialization
+          # because here a parse error is due to a bad message rather than an
+          # incompatible `self.serializer`.
+          throw :invalid_message_format, error
+        end
+
+        def serialize_to_json_safe_string(data)
+          encode(serialize(data), url_safe: false)
+        end
+
+        def deserialize_from_json_safe_string(string)
+          deserialize(decode(string, url_safe: false))
+        end
     end
   end
 end

data/lib/active_support/messages/rotation_configuration.rb

@@ -9,7 +9,8 @@ module ActiveSupport
         @signed, @encrypted = [], []
       end
 
-      def rotate(kind, *args)
+      def rotate(kind, *args, **options)
+        args << options unless options.empty?
         case kind
         when :signed
           @signed << args

data/lib/active_support/messages/rotation_coordinator.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/slice"
+
+module ActiveSupport
+  module Messages
+    class RotationCoordinator # :nodoc:
+      attr_accessor :transitional
+
+      def initialize(&secret_generator)
+        raise ArgumentError, "A secret generator block is required" unless secret_generator
+        @secret_generator = secret_generator
+        @rotate_options = []
+        @on_rotation = nil
+        @codecs = {}
+      end
+
+      def [](salt)
+        @codecs[salt] ||= build_with_rotations(salt)
+      end
+
+      def []=(salt, codec)
+        @codecs[salt] = codec
+      end
+
+      def rotate(**options, &block)
+        raise ArgumentError, "Options cannot be specified when using a block" if block && !options.empty?
+        changing_configuration!
+
+        @rotate_options << (block || options)
+
+        self
+      end
+
+      def rotate_defaults
+        rotate()
+      end
+
+      def clear_rotations
+        changing_configuration!
+        @rotate_options.clear
+        self
+      end
+
+      def on_rotation(&callback)
+        changing_configuration!
+        @on_rotation = callback
+      end
+
+      private
+        def changing_configuration!
+          if @codecs.any?
+            raise <<~MESSAGE
+              Cannot change #{self.class} configuration after it has already been applied.
+
+              The configuration has been applied with the following salts:
+              #{@codecs.keys.map { |salt| "- #{salt.inspect}" }.join("\n")}
+            MESSAGE
+          end
+        end
+
+        def normalize_options(options)
+          options = options.dup
+
+          options[:secret_generator] ||= @secret_generator
+
+          secret_generator_kwargs = options[:secret_generator].parameters.
+            filter_map { |type, name| name if type == :key || type == :keyreq }
+          options[:secret_generator_options] = options.extract!(*secret_generator_kwargs)
+
+          options[:on_rotation] = @on_rotation
+
+          options
+        end
+
+        def build_with_rotations(salt)
+          rotate_options = @rotate_options.map { |options| options.is_a?(Proc) ? options.(salt) : options }
+          transitional = self.transitional && rotate_options.first
+          rotate_options.compact!
+          rotate_options[0..1] = rotate_options[0..1].reverse if transitional
+          rotate_options = rotate_options.map { |options| normalize_options(options) }.uniq
+
+          raise "No options have been configured for #{salt}" if rotate_options.empty?
+
+          rotate_options.map { |options| build(salt.to_s, **options) }.reduce(&:fall_back_to)
+        end
+
+        def build(salt, secret_generator:, secret_generator_options:, **options)
+          raise NotImplementedError
+        end
+    end
+  end
+end

data/lib/active_support/messages/rotator.rb

@@ -3,53 +3,56 @@
 module ActiveSupport
   module Messages
     module Rotator # :nodoc:
-      def initialize(*, **options)
-        super
-
-        @options   = options
+      def initialize(*args, on_rotation: nil, **options)
+        super(*args, **options)
+        @args = args
+        @options = options
         @rotations = []
+        @on_rotation = on_rotation
       end
 
-      def rotate(*secrets, **options)
-        @rotations << build_rotation(*secrets, @options.merge(options))
+      def rotate(*args, **options)
+        fall_back_to build_rotation(*args, **options)
       end
 
-      module Encryptor
-        include Rotator
-
-        def decrypt_and_verify(*args, on_rotation: nil, **options)
-          super
-        rescue MessageEncryptor::InvalidMessage, MessageVerifier::InvalidSignature
-          run_rotations(on_rotation) { |encryptor| encryptor.decrypt_and_verify(*args, **options) } || raise
-        end
+      def fall_back_to(fallback)
+        @rotations << fallback
+        self
+      end
 
-        private
-          def build_rotation(secret = @secret, sign_secret = @sign_secret, options)
-            self.class.new(secret, sign_secret, **options)
+      def read_message(message, on_rotation: @on_rotation, **options)
+        if @rotations.empty?
+          super(message, **options)
+        else
+          thrown, error = catch_rotation_error do
+            return super(message, **options)
           end
-      end
 
-      module Verifier
-        include Rotator
+          @rotations.each do |rotation|
+            catch_rotation_error do
+              value = rotation.read_message(message, **options)
+              on_rotation&.call
+              return value
+            end
+          end
 
-        def verified(*args, on_rotation: nil, **options)
-          super || run_rotations(on_rotation) { |verifier| verifier.verified(*args, **options) }
+          throw thrown, error
         end
-
-        private
-          def build_rotation(secret = @secret, options)
-            self.class.new(secret, **options)
-          end
       end
 
       private
-        def run_rotations(on_rotation)
-          @rotations.find do |rotation|
-            if message = yield(rotation) rescue next
-              on_rotation.call if on_rotation
-              return message
+        def build_rotation(*args, **options)
+          self.class.new(*args, *@args.drop(args.length), **@options, **options)
+        end
+
+        def catch_rotation_error(&block)
+          error = catch :invalid_message_format do
+            error = catch :invalid_message_serialization do
+              return [nil, block.call]
             end
+            return [:invalid_message_serialization, error]
           end
+          [:invalid_message_format, error]
         end
     end
   end

data/lib/active_support/messages/serializer_with_fallback.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/kernel/reporting"
+require "active_support/notifications"
+
+module ActiveSupport
+  module Messages # :nodoc:
+    module SerializerWithFallback # :nodoc:
+      def self.[](format)
+        if format.to_s.include?("message_pack") && !defined?(ActiveSupport::MessagePack)
+          require "active_support/message_pack"
+        end
+
+        SERIALIZERS.fetch(format)
+      end
+
+      def load(dumped)
+        format = detect_format(dumped)
+
+        if format == self.format
+          _load(dumped)
+        elsif format && fallback?(format)
+          payload = { serializer: SERIALIZERS.key(self), fallback: format, serialized: dumped }
+          ActiveSupport::Notifications.instrument("message_serializer_fallback.active_support", payload) do
+            payload[:deserialized] = SERIALIZERS[format]._load(dumped)
+          end
+        else
+          raise "Unsupported serialization format"
+        end
+      end
+
+      private
+        def detect_format(dumped)
+          case
+          when MessagePackWithFallback.dumped?(dumped)
+            :message_pack
+          when MarshalWithFallback.dumped?(dumped)
+            :marshal
+          when JsonWithFallback.dumped?(dumped)
+            :json
+          end
+        end
+
+        def fallback?(format)
+          format != :marshal
+        end
+
+        module AllowMarshal
+          private
+            def fallback?(format)
+              super || format == :marshal
+            end
+        end
+
+        module MarshalWithFallback
+          include SerializerWithFallback
+          extend self
+
+          def format
+            :marshal
+          end
+
+          def dump(object)
+            Marshal.dump(object)
+          end
+
+          def _load(dumped)
+            Marshal.load(dumped)
+          end
+
+          MARSHAL_SIGNATURE = "\x04\x08"
+
+          def dumped?(dumped)
+            dumped.start_with?(MARSHAL_SIGNATURE)
+          end
+        end
+
+        module JsonWithFallback
+          include SerializerWithFallback
+          extend self
+
+          def format
+            :json
+          end
+
+          def dump(object)
+            ActiveSupport::JSON.encode(object)
+          end
+
+          def _load(dumped)
+            ActiveSupport::JSON.decode(dumped)
+          end
+
+          JSON_START_WITH = /\A(?:[{\["]|-?\d|true|false|null)/
+
+          def dumped?(dumped)
+            JSON_START_WITH.match?(dumped)
+          end
+
+          private
+            def detect_format(dumped)
+              # Assume JSON format if format could not be determined.
+              super || :json
+            end
+        end
+
+        module JsonWithFallbackAllowMarshal
+          include JsonWithFallback
+          include AllowMarshal
+          extend self
+        end
+
+        module MessagePackWithFallback
+          include SerializerWithFallback
+          extend self
+
+          def format
+            :message_pack
+          end
+
+          def dump(object)
+            ActiveSupport::MessagePack.dump(object)
+          end
+
+          def _load(dumped)
+            ActiveSupport::MessagePack.load(dumped)
+          end
+
+          def dumped?(dumped)
+            available? && ActiveSupport::MessagePack.signature?(dumped)
+          end
+
+          private
+            def available?
+              return @available if defined?(@available)
+              silence_warnings { require "active_support/message_pack" }
+              @available = true
+            rescue LoadError
+              @available = false
+            end
+        end
+
+        module MessagePackWithFallbackAllowMarshal
+          include MessagePackWithFallback
+          include AllowMarshal
+          extend self
+        end
+
+        SERIALIZERS = {
+          marshal: MarshalWithFallback,
+          json: JsonWithFallback,
+          json_allow_marshal: JsonWithFallbackAllowMarshal,
+          message_pack: MessagePackWithFallback,
+          message_pack_allow_marshal: MessagePackWithFallbackAllowMarshal,
+        }
+    end
+  end
+end

data/lib/active_support/multibyte/chars.rb

@@ -5,8 +5,10 @@ require "active_support/core_ext/string/access"
 require "active_support/core_ext/string/behavior"
 require "active_support/core_ext/module/delegation"
 
-module ActiveSupport #:nodoc:
-  module Multibyte #:nodoc:
+module ActiveSupport # :nodoc:
+  module Multibyte # :nodoc:
+    # = Active Support \Multibyte \Chars
+    #
     # Chars enables you to work transparently with UTF-8 encoding in the Ruby
     # String class without having extensive knowledge about the encoding. A
     # Chars object accepts a string upon initialization and proxies String
@@ -48,7 +50,7 @@ module ActiveSupport #:nodoc:
       alias to_s wrapped_string
       alias to_str wrapped_string
 
-      delegate :<=>, :=~, :acts_like_string?, to: :wrapped_string
+      delegate :<=>, :=~, :match?, :acts_like_string?, to: :wrapped_string
 
       # Creates a new Chars instance by wrapping _string_.
       def initialize(string)
@@ -59,7 +61,7 @@ module ActiveSupport #:nodoc:
       # Forward all undefined methods to the wrapped string.
       def method_missing(method, *args, &block)
         result = @wrapped_string.__send__(method, *args, &block)
-        if /!$/.match?(method)
+        if method.end_with?("!")
           self if result
         else
           result.kind_of?(String) ? chars(result) : result
@@ -73,17 +75,6 @@ module ActiveSupport #:nodoc:
         @wrapped_string.respond_to?(method, include_private)
       end
 
-      # Returns +true+ when the proxy class can handle the string. Returns
-      # +false+ otherwise.
-      def self.consumes?(string)
-        ActiveSupport::Deprecation.warn(<<-MSG.squish)
-          ActiveSupport::Multibyte::Chars.consumes? is deprecated and will be
-          removed from Rails 6.1. Use string.is_utf8? instead.
-        MSG
-
-        string.encoding == Encoding::UTF_8
-      end
-
       # Works just like <tt>String#split</tt>, with the exception that the items
       # in the resulting list are Chars instances instead of String. This makes
       # chaining methods easier.
@@ -113,7 +104,7 @@ module ActiveSupport #:nodoc:
       #
       #   'Café'.mb_chars.reverse.to_s # => 'éfaC'
       def reverse
-        chars(@wrapped_string.scan(/\X/).reverse.join)
+        chars(@wrapped_string.grapheme_clusters.reverse.join)
       end
 
       # Limits the byte size of the string to a number of bytes without breaking
@@ -134,46 +125,18 @@ module ActiveSupport #:nodoc:
       end
       alias_method :titlecase, :titleize
 
-      # Returns the KC normalization of the string by default. NFKC is
-      # considered the best normalization form for passing strings to databases
-      # and validations.
-      #
-      # * <tt>form</tt> - The form you want to normalize in. Should be one of the following:
-      #   <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
-      #   ActiveSupport::Multibyte::Unicode.default_normalization_form
-      def normalize(form = nil)
-        form ||= Unicode.default_normalization_form
-
-        # See https://www.unicode.org/reports/tr15, Table 1
-        if alias_form = Unicode::NORMALIZATION_FORM_ALIASES[form]
-          ActiveSupport::Deprecation.warn(<<-MSG.squish)
-            ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
-            removed from Rails 6.1. Use #unicode_normalize(:#{alias_form}) instead.
-          MSG
-
-          send(:unicode_normalize, alias_form)
-        else
-          ActiveSupport::Deprecation.warn(<<-MSG.squish)
-            ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
-            removed from Rails 6.1. Use #unicode_normalize instead.
-          MSG
-
-          raise ArgumentError, "#{form} is not a valid normalization variant", caller
-        end
-      end
-
       # Performs canonical decomposition on all the characters.
       #
-      #   'é'.length                         # => 2
-      #   'é'.mb_chars.decompose.to_s.length # => 3
+      #   'é'.length                         # => 1
+      #   'é'.mb_chars.decompose.to_s.length # => 2
       def decompose
         chars(Unicode.decompose(:canonical, @wrapped_string.codepoints.to_a).pack("U*"))
       end
 
       # Performs composition on all the characters.
       #
-      #   'é'.length                       # => 3
-      #   'é'.mb_chars.compose.to_s.length # => 2
+      #   'é'.length                       # => 1
+      #   'é'.mb_chars.compose.to_s.length # => 1
       def compose
         chars(Unicode.compose(@wrapped_string.codepoints.to_a).pack("U*"))
       end
@@ -181,9 +144,9 @@ module ActiveSupport #:nodoc:
       # Returns the number of grapheme clusters in the string.
       #
       #   'क्षि'.mb_chars.length   # => 4
-      #   'क्षि'.mb_chars.grapheme_length # => 3
+      #   'क्षि'.mb_chars.grapheme_length # => 2
       def grapheme_length
-        @wrapped_string.scan(/\X/).length
+        @wrapped_string.grapheme_clusters.length
       end
 
       # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent
@@ -195,13 +158,13 @@ module ActiveSupport #:nodoc:
         chars(Unicode.tidy_bytes(@wrapped_string, force))
       end
 
-      def as_json(options = nil) #:nodoc:
+      def as_json(options = nil) # :nodoc:
         to_s.as_json(options)
       end
 
       %w(reverse tidy_bytes).each do |method|
         define_method("#{method}!") do |*args|
-          @wrapped_string = send(method, *args).to_s
+          @wrapped_string = public_send(method, *args).to_s
           self
         end
       end