activesupport 7.0.7.2 → 7.1.0.beta1

data/lib/active_support/messages/metadata.rb

@@ -1,80 +1,146 @@
 # frozen_string_literal: true
 
 require "time"
+require "active_support/json"
+require_relative "serializer_with_fallback"
 
 module ActiveSupport
   module Messages # :nodoc:
-    class Metadata # :nodoc:
-      def initialize(message, expires_at = nil, purpose = nil)
-        @message, @purpose = message, purpose
-        @expires_at = expires_at.is_a?(String) ? parse_expires_at(expires_at) : expires_at
-      end
+    module Metadata # :nodoc:
+      singleton_class.attr_accessor :use_message_serializer_for_metadata
+
+      ENVELOPE_SERIALIZERS = [
+        *SerializerWithFallback::SERIALIZERS.values,
+        ActiveSupport::JSON,
+        ::JSON,
+        Marshal,
+      ]
+
+      TIMESTAMP_SERIALIZERS = [
+        SerializerWithFallback::SERIALIZERS.fetch(:message_pack),
+        SerializerWithFallback::SERIALIZERS.fetch(:message_pack_allow_marshal),
+      ]
 
-      def as_json(options = {})
-        { _rails: { message: @message, exp: @expires_at, pur: @purpose } }
+      ActiveSupport.on_load(:message_pack) do
+        ENVELOPE_SERIALIZERS << ActiveSupport::MessagePack
+        TIMESTAMP_SERIALIZERS << ActiveSupport::MessagePack
       end
 
-      class << self
-        def wrap(message, expires_at: nil, expires_in: nil, purpose: nil)
-          if expires_at || expires_in || purpose
-            JSON.encode new(encode(message), pick_expiry(expires_at, expires_in), purpose)
+      private
+        def serialize_with_metadata(data, **metadata)
+          has_metadata = metadata.any? { |k, v| v }
+
+          if has_metadata && !use_message_serializer_for_metadata?
+            data_string = serialize_to_json_safe_string(data)
+            envelope = wrap_in_metadata_legacy_envelope({ "message" => data_string }, **metadata)
+            serialize_to_json(envelope)
           else
-            message
+            data = wrap_in_metadata_envelope({ "data" => data }, **metadata) if has_metadata
+            serialize(data)
           end
         end
 
-        def verify(message, purpose)
-          extract_metadata(message).verify(purpose)
-        end
-
-        private
-          def pick_expiry(expires_at, expires_in)
-            if expires_at
-              expires_at.utc.iso8601(3)
-            elsif expires_in
-              Time.now.utc.advance(seconds: expires_in).iso8601(3)
+        def deserialize_with_metadata(message, **expected_metadata)
+          if dual_serialized_metadata_envelope_json?(message)
+            envelope = deserialize_from_json(message)
+            extracted = extract_from_metadata_envelope(envelope, **expected_metadata)
+            deserialize_from_json_safe_string(extracted["message"])
+          else
+            deserialized = deserialize(message)
+            if metadata_envelope?(deserialized)
+              extract_from_metadata_envelope(deserialized, **expected_metadata)["data"]
+            elsif expected_metadata.none? { |k, v| v }
+              deserialized
+            else
+              throw :invalid_message_content, "missing metadata"
             end
           end
+        end
 
-          def extract_metadata(message)
-            data = JSON.decode(message) rescue nil
+        def use_message_serializer_for_metadata?
+          Metadata.use_message_serializer_for_metadata && Metadata::ENVELOPE_SERIALIZERS.include?(serializer)
+        end
 
-            if data.is_a?(Hash) && data.key?("_rails")
-              new(decode(data["_rails"]["message"]), data["_rails"]["exp"], data["_rails"]["pur"])
-            else
-              new(message)
-            end
-          end
+        def wrap_in_metadata_envelope(hash, expires_at: nil, expires_in: nil, purpose: nil)
+          expiry = pick_expiry(expires_at, expires_in)
+          hash["exp"] = expiry if expiry
+          hash["pur"] = purpose.to_s if purpose
+          { "_rails" => hash }
+        end
+
+        def wrap_in_metadata_legacy_envelope(hash, expires_at: nil, expires_in: nil, purpose: nil)
+          expiry = pick_expiry(expires_at, expires_in)
+          hash["exp"] = expiry
+          hash["pur"] = purpose
+          { "_rails" => hash }
+        end
+
+        def extract_from_metadata_envelope(envelope, purpose: nil)
+          hash = envelope["_rails"]
 
-          def encode(message)
-            ::Base64.strict_encode64(message)
+          if hash["exp"] && Time.now.utc >= parse_expiry(hash["exp"])
+            throw :invalid_message_content, "expired"
           end
 
-          def decode(message)
-            ::Base64.strict_decode64(message)
+          if hash["pur"] != purpose&.to_s
+            throw :invalid_message_content, "mismatched purpose"
           end
-      end
 
-      def verify(purpose)
-        @message if match?(purpose) && fresh?
-      end
+          hash
+        end
 
-      private
-        def match?(purpose)
-          @purpose.to_s == purpose.to_s
+        def metadata_envelope?(object)
+          object.is_a?(Hash) && object.key?("_rails")
         end
 
-        def fresh?
-          @expires_at.nil? || Time.now.utc < @expires_at
+        def dual_serialized_metadata_envelope_json?(string)
+          string.start_with?('{"_rails":{"message":')
+        end
+
+        def pick_expiry(expires_at, expires_in)
+          expiry = if expires_at
+            expires_at.utc
+          elsif expires_in
+            Time.now.utc.advance(seconds: expires_in)
+          end
+
+          unless Metadata::TIMESTAMP_SERIALIZERS.include?(serializer)
+            expiry = expiry&.iso8601(3)
+          end
+
+          expiry
         end
 
-        def parse_expires_at(expires_at)
-          if ActiveSupport.use_standard_json_time_format
+        def parse_expiry(expires_at)
+          if !expires_at.is_a?(String)
+            expires_at
+          elsif ActiveSupport.use_standard_json_time_format
             Time.iso8601(expires_at)
           else
             Time.parse(expires_at)
           end
         end
+
+        def serialize_to_json(data)
+          ActiveSupport::JSON.encode(data)
+        end
+
+        def deserialize_from_json(serialized)
+          ActiveSupport::JSON.decode(serialized)
+        rescue ::JSON::ParserError => error
+          # Throw :invalid_message_format instead of :invalid_message_serialization
+          # because here a parse error is due to a bad message rather than an
+          # incompatible `self.serializer`.
+          throw :invalid_message_format, error
+        end
+
+        def serialize_to_json_safe_string(data)
+          encode(serialize(data), url_safe: false)
+        end
+
+        def deserialize_from_json_safe_string(string)
+          deserialize(decode(string, url_safe: false))
+        end
     end
   end
 end

data/lib/active_support/messages/rotation_coordinator.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/slice"
+
+module ActiveSupport
+  module Messages
+    class RotationCoordinator # :nodoc:
+      attr_accessor :transitional
+
+      def initialize(&secret_generator)
+        raise ArgumentError, "A secret generator block is required" unless secret_generator
+        @secret_generator = secret_generator
+        @rotate_options = []
+        @on_rotation = nil
+        @codecs = {}
+      end
+
+      def [](salt)
+        @codecs[salt] ||= build_with_rotations(salt)
+      end
+
+      def []=(salt, codec)
+        @codecs[salt] = codec
+      end
+
+      def rotate(**options, &block)
+        raise ArgumentError, "Options cannot be specified when using a block" if block && !options.empty?
+        changing_configuration!
+
+        @rotate_options << (block || options)
+
+        self
+      end
+
+      def rotate_defaults
+        rotate()
+      end
+
+      def clear_rotations
+        changing_configuration!
+        @rotate_options.clear
+        self
+      end
+
+      def on_rotation(&callback)
+        changing_configuration!
+        @on_rotation = callback
+      end
+
+      private
+        def changing_configuration!
+          if @codecs.any?
+            raise <<~MESSAGE
+              Cannot change #{self.class} configuration after it has already been applied.
+
+              The configuration has been applied with the following salts:
+              #{@codecs.keys.map { |salt| "- #{salt.inspect}" }.join("\n")}
+            MESSAGE
+          end
+        end
+
+        def normalize_options(options)
+          options = options.dup
+
+          options[:secret_generator] ||= @secret_generator
+
+          secret_generator_kwargs = options[:secret_generator].parameters.
+            filter_map { |type, name| name if type == :key || type == :keyreq }
+          options[:secret_generator_options] = options.extract!(*secret_generator_kwargs)
+
+          options[:on_rotation] = @on_rotation
+
+          options
+        end
+
+        def build_with_rotations(salt)
+          rotate_options = @rotate_options.map { |options| options.is_a?(Proc) ? options.(salt) : options }
+          transitional = self.transitional && rotate_options.first
+          rotate_options.compact!
+          rotate_options[0..1] = rotate_options[0..1].reverse if transitional
+          rotate_options = rotate_options.map { |options| normalize_options(options) }.uniq
+
+          raise "No options have been configured for #{salt}" if rotate_options.empty?
+
+          rotate_options.map { |options| build(salt.to_s, **options) }.reduce(&:fall_back_to)
+        end
+
+        def build(salt, secret_generator:, secret_generator_options:, **options)
+          raise NotImplementedError
+        end
+    end
+  end
+end

data/lib/active_support/messages/rotator.rb

@@ -3,54 +3,56 @@
 module ActiveSupport
   module Messages
     module Rotator # :nodoc:
-      def initialize(*secrets, on_rotation: nil, **options)
-        super(*secrets, **options)
-
-        @options   = options
+      def initialize(*args, on_rotation: nil, **options)
+        super(*args, **options)
+        @args = args
+        @options = options
         @rotations = []
         @on_rotation = on_rotation
       end
 
-      def rotate(*secrets, **options)
-        @rotations << build_rotation(*secrets, @options.merge(options))
+      def rotate(*args, **options)
+        fall_back_to build_rotation(*args, **options)
       end
 
-      module Encryptor
-        include Rotator
-
-        def decrypt_and_verify(*args, on_rotation: @on_rotation, **options)
-          super
-        rescue MessageEncryptor::InvalidMessage, MessageVerifier::InvalidSignature
-          run_rotations(on_rotation) { |encryptor| encryptor.decrypt_and_verify(*args, **options) } || raise
-        end
+      def fall_back_to(fallback)
+        @rotations << fallback
+        self
+      end
 
-        private
-          def build_rotation(secret = @secret, sign_secret = @sign_secret, options)
-            self.class.new(secret, sign_secret, **options)
+      def read_message(message, on_rotation: @on_rotation, **options)
+        if @rotations.empty?
+          super(message, **options)
+        else
+          thrown, error = catch_rotation_error do
+            return super(message, **options)
           end
-      end
 
-      module Verifier
-        include Rotator
+          @rotations.each do |rotation|
+            catch_rotation_error do
+              value = rotation.read_message(message, **options)
+              on_rotation&.call
+              return value
+            end
+          end
 
-        def verified(*args, on_rotation: @on_rotation, **options)
-          super || run_rotations(on_rotation) { |verifier| verifier.verified(*args, **options) }
+          throw thrown, error
         end
-
-        private
-          def build_rotation(secret = @secret, options)
-            self.class.new(secret, **options)
-          end
       end
 
       private
-        def run_rotations(on_rotation)
-          @rotations.find do |rotation|
-            if message = yield(rotation) rescue next
-              on_rotation&.call
-              return message
+        def build_rotation(*args, **options)
+          self.class.new(*args, *@args.drop(args.length), **@options, **options)
+        end
+
+        def catch_rotation_error(&block)
+          error = catch :invalid_message_format do
+            error = catch :invalid_message_serialization do
+              return [nil, block.call]
             end
+            return [:invalid_message_serialization, error]
           end
+          [:invalid_message_format, error]
         end
     end
   end

data/lib/active_support/messages/serializer_with_fallback.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/kernel/reporting"
+require "active_support/notifications"
+
+module ActiveSupport
+  module Messages # :nodoc:
+    module SerializerWithFallback # :nodoc:
+      def self.[](format)
+        if format.to_s.include?("message_pack") && !defined?(ActiveSupport::MessagePack)
+          require "active_support/message_pack"
+        end
+
+        SERIALIZERS.fetch(format)
+      end
+
+      def load(dumped)
+        format = detect_format(dumped)
+
+        if format == self.format
+          _load(dumped)
+        elsif format && fallback?(format)
+          payload = { serializer: SERIALIZERS.key(self), fallback: format, serialized: dumped }
+          ActiveSupport::Notifications.instrument("message_serializer_fallback.active_support", payload) do
+            payload[:deserialized] = SERIALIZERS[format]._load(dumped)
+          end
+        else
+          raise "Unsupported serialization format"
+        end
+      end
+
+      private
+        def detect_format(dumped)
+          case
+          when MessagePackWithFallback.dumped?(dumped)
+            :message_pack
+          when MarshalWithFallback.dumped?(dumped)
+            :marshal
+          when JsonWithFallback.dumped?(dumped)
+            :json
+          end
+        end
+
+        def fallback?(format)
+          format != :marshal
+        end
+
+        module AllowMarshal
+          private
+            def fallback?(format)
+              super || format == :marshal
+            end
+        end
+
+        module MarshalWithFallback
+          include SerializerWithFallback
+          extend self
+
+          def format
+            :marshal
+          end
+
+          def dump(object)
+            Marshal.dump(object)
+          end
+
+          def _load(dumped)
+            Marshal.load(dumped)
+          end
+
+          MARSHAL_SIGNATURE = "\x04\x08"
+
+          def dumped?(dumped)
+            dumped.start_with?(MARSHAL_SIGNATURE)
+          end
+        end
+
+        module JsonWithFallback
+          include SerializerWithFallback
+          extend self
+
+          def format
+            :json
+          end
+
+          def dump(object)
+            ActiveSupport::JSON.encode(object)
+          end
+
+          def _load(dumped)
+            ActiveSupport::JSON.decode(dumped)
+          end
+
+          JSON_START_WITH = /\A(?:[{\["]|-?\d|true|false|null)/
+
+          def dumped?(dumped)
+            JSON_START_WITH.match?(dumped)
+          end
+
+          private
+            def detect_format(dumped)
+              # Assume JSON format if format could not be determined.
+              super || :json
+            end
+        end
+
+        module JsonWithFallbackAllowMarshal
+          include JsonWithFallback
+          include AllowMarshal
+          extend self
+        end
+
+        module MessagePackWithFallback
+          include SerializerWithFallback
+          extend self
+
+          def format
+            :message_pack
+          end
+
+          def dump(object)
+            ActiveSupport::MessagePack.dump(object)
+          end
+
+          def _load(dumped)
+            ActiveSupport::MessagePack.load(dumped)
+          end
+
+          def dumped?(dumped)
+            available? && ActiveSupport::MessagePack.signature?(dumped)
+          end
+
+          private
+            def available?
+              return @available if defined?(@available)
+              silence_warnings { require "active_support/message_pack" }
+              @available = true
+            rescue LoadError
+              @available = false
+            end
+        end
+
+        module MessagePackWithFallbackAllowMarshal
+          include MessagePackWithFallback
+          include AllowMarshal
+          extend self
+        end
+
+        SERIALIZERS = {
+          marshal: MarshalWithFallback,
+          json: JsonWithFallback,
+          json_allow_marshal: JsonWithFallbackAllowMarshal,
+          message_pack: MessagePackWithFallback,
+          message_pack_allow_marshal: MessagePackWithFallbackAllowMarshal,
+        }
+    end
+  end
+end

data/lib/active_support/multibyte/chars.rb

@@ -7,6 +7,8 @@ require "active_support/core_ext/module/delegation"
 
 module ActiveSupport # :nodoc:
   module Multibyte # :nodoc:
+    # = Active Support \Multibyte \Chars
+    #
     # Chars enables you to work transparently with UTF-8 encoding in the Ruby
     # String class without having extensive knowledge about the encoding. A
     # Chars object accepts a string upon initialization and proxies String

data/lib/active_support/multibyte/unicode.rb

@@ -22,43 +22,15 @@ module ActiveSupport
         codepoints.pack("U*").unicode_normalize(:nfc).codepoints
       end
 
-      # Rubinius' String#scrub, however, doesn't support ASCII-incompatible chars.
-      if !defined?(Rubinius)
-        # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent
-        # resulting in a valid UTF-8 string.
-        #
-        # Passing +true+ will forcibly tidy all bytes, assuming that the string's
-        # encoding is entirely CP1252 or ISO-8859-1.
-        def tidy_bytes(string, force = false)
-          return string if string.empty? || string.ascii_only?
-          return recode_windows1252_chars(string) if force
-          string.scrub { |bad| recode_windows1252_chars(bad) }
-        end
-      else
-        def tidy_bytes(string, force = false)
-          return string if string.empty?
-          return recode_windows1252_chars(string) if force
-
-          # We can't transcode to the same format, so we choose a nearly-identical encoding.
-          # We're going to 'transcode' bytes from UTF-8 when possible, then fall back to
-          # CP1252 when we get errors. The final string will be 'converted' back to UTF-8
-          # before returning.
-          reader = Encoding::Converter.new(Encoding::UTF_8, Encoding::UTF_16LE)
-
-          source = string.dup
-          out = "".force_encoding(Encoding::UTF_16LE)
-
-          loop do
-            reader.primitive_convert(source, out)
-            _, _, _, error_bytes, _ = reader.primitive_errinfo
-            break if error_bytes.nil?
-            out << error_bytes.encode(Encoding::UTF_16LE, Encoding::Windows_1252, invalid: :replace, undef: :replace)
-          end
-
-          reader.finish
-
-          out.encode!(Encoding::UTF_8)
-        end
+      # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent
+      # resulting in a valid UTF-8 string.
+      #
+      # Passing +true+ will forcibly tidy all bytes, assuming that the string's
+      # encoding is entirely CP1252 or ISO-8859-1.
+      def tidy_bytes(string, force = false)
+        return string if string.empty? || string.ascii_only?
+        return recode_windows1252_chars(string) if force
+        string.scrub { |bad| recode_windows1252_chars(bad) }
       end
 
       private