activesupport 5.2.7 → 6.0.0.beta1

data/lib/active_support/logger_thread_safe_level.rb

@@ -1,34 +1,55 @@
 # frozen_string_literal: true
 
 require "active_support/concern"
-require "fiber"
+require "active_support/core_ext/module/attribute_accessors"
+require "concurrent"
 
 module ActiveSupport
   module LoggerThreadSafeLevel # :nodoc:
     extend ActiveSupport::Concern
 
+    included do
+      cattr_accessor :local_levels, default: Concurrent::Map.new(initial_capacity: 2), instance_accessor: false
+    end
+
+    Logger::Severity.constants.each do |severity|
+      class_eval(<<-EOT, __FILE__, __LINE__ + 1)
+        def #{severity.downcase}?                # def debug?
+          Logger::#{severity} >= level           #   DEBUG >= level
+        end                                      # end
+      EOT
+    end
+
     def after_initialize
-      @local_levels = Concurrent::Map.new(initial_capacity: 2)
+      ActiveSupport::Deprecation.warn(
+        "Logger don't need to call #after_initialize directly anymore. It will be deprecated without replacement in " \
+        "Rails 6.1."
+      )
     end
 
     def local_log_id
-      Fiber.current.__id__
+      Thread.current.__id__
     end
 
     def local_level
-      @local_levels[local_log_id]
+      self.class.local_levels[local_log_id]
     end
 
     def local_level=(level)
       if level
-        @local_levels[local_log_id] = level
+        self.class.local_levels[local_log_id] = level
       else
-        @local_levels.delete(local_log_id)
+        self.class.local_levels.delete(local_log_id)
       end
     end
 
     def level
       local_level || super
     end
+
+    def add(severity, message = nil, progname = nil, &block) # :nodoc:
+      return true if @logdev.nil? || (severity || UNKNOWN) < level
+      super
+    end
   end
 end

data/lib/active_support/message_encryptor.rb

@@ -182,7 +182,7 @@ module ActiveSupport
 
       def _decrypt(encrypted_message, purpose)
         cipher = new_cipher
-        encrypted_data, iv, auth_tag = encrypted_message.split("--".freeze).map { |v| ::Base64.strict_decode64(v) }
+        encrypted_data, iv, auth_tag = encrypted_message.split("--").map { |v| ::Base64.strict_decode64(v) }
 
         # Currently the OpenSSL bindings do not raise an error if auth_tag is
         # truncated, which would allow an attacker to easily forge it. See
@@ -210,9 +210,7 @@ module ActiveSupport
         OpenSSL::Cipher.new(@cipher)
       end
 
-      def verifier
-        @verifier
-      end
+      attr_reader :verifier
 
       def aead_mode?
         @aead_mode ||= new_cipher.authenticated?

data/lib/active_support/message_verifier.rb

@@ -122,7 +122,7 @@ module ActiveSupport
     def valid_message?(signed_message)
       return if signed_message.nil? || !signed_message.valid_encoding? || signed_message.blank?
 
-      data, digest = signed_message.split("--".freeze)
+      data, digest = signed_message.split("--")
       data.present? && digest.present? && ActiveSupport::SecurityUtils.secure_compare(digest, generate_digest(data))
     end
 
@@ -150,7 +150,7 @@ module ActiveSupport
     def verified(signed_message, purpose: nil, **)
       if valid_message?(signed_message)
         begin
-          data = signed_message.split("--".freeze)[0]
+          data = signed_message.split("--")[0]
           message = Messages::Metadata.verify(decode(data), purpose)
           @serializer.load(message) if message
         rescue ArgumentError => argument_error

data/lib/active_support/multibyte/chars.rb

@@ -4,7 +4,6 @@ require "active_support/json"
 require "active_support/core_ext/string/access"
 require "active_support/core_ext/string/behavior"
 require "active_support/core_ext/module/delegation"
-require "active_support/core_ext/regexp"
 
 module ActiveSupport #:nodoc:
   module Multibyte #:nodoc:
@@ -18,7 +17,7 @@ module ActiveSupport #:nodoc:
     # through the +mb_chars+ method. Methods which would normally return a
     # String object now return a Chars object so methods can be chained.
     #
-    #   'The Perfect String  '.mb_chars.downcase.strip.normalize
+    #   'The Perfect String  '.mb_chars.downcase.strip
     #   # => #<ActiveSupport::Multibyte::Chars:0x007fdc434ccc10 @wrapped_string="the perfect string">
     #
     # Chars objects are perfectly interchangeable with String objects as long as
@@ -77,6 +76,11 @@ module ActiveSupport #:nodoc:
       # Returns +true+ when the proxy class can handle the string. Returns
       # +false+ otherwise.
       def self.consumes?(string)
+        ActiveSupport::Deprecation.warn(<<-MSG.squish)
+          ActiveSupport::Multibyte::Chars.consumes? is deprecated and will be
+          removed from Rails 6.1. Use string.is_utf8? instead.
+        MSG
+
         string.encoding == Encoding::UTF_8
       end
 
@@ -109,7 +113,7 @@ module ActiveSupport #:nodoc:
       #
       #   'Café'.mb_chars.reverse.to_s # => 'éfaC'
       def reverse
-        chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack("U*"))
+        chars(@wrapped_string.scan(/\X/).reverse.join)
       end
 
       # Limits the byte size of the string to a number of bytes without breaking
@@ -118,35 +122,7 @@ module ActiveSupport #:nodoc:
       #
       #   'こんにちは'.mb_chars.limit(7).to_s # => "こん"
       def limit(limit)
-        slice(0...translate_offset(limit))
-      end
-
-      # Converts characters in the string to uppercase.
-      #
-      #   'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?"
-      def upcase
-        chars Unicode.upcase(@wrapped_string)
-      end
-
-      # Converts characters in the string to lowercase.
-      #
-      #   'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum"
-      def downcase
-        chars Unicode.downcase(@wrapped_string)
-      end
-
-      # Converts characters in the string to the opposite case.
-      #
-      #    'El Cañón'.mb_chars.swapcase.to_s # => "eL cAÑÓN"
-      def swapcase
-        chars Unicode.swapcase(@wrapped_string)
-      end
-
-      # Converts the first character to uppercase and the remainder to lowercase.
-      #
-      #  'über'.mb_chars.capitalize.to_s # => "Über"
-      def capitalize
-        (slice(0) || chars("")).upcase + (slice(1..-1) || chars("")).downcase
+        truncate_bytes(limit, omission: nil)
       end
 
       # Capitalizes the first letter of every word, when possible.
@@ -154,7 +130,7 @@ module ActiveSupport #:nodoc:
       #   "ÉL QUE SE ENTERÓ".mb_chars.titleize.to_s    # => "Él Que Se Enteró"
       #   "日本語".mb_chars.titleize.to_s               # => "日本語"
       def titleize
-        chars(downcase.to_s.gsub(/\b('?\S)/u) { Unicode.upcase($1) })
+        chars(downcase.to_s.gsub(/\b('?\S)/u) { $1.upcase })
       end
       alias_method :titlecase, :titleize
 
@@ -166,7 +142,24 @@ module ActiveSupport #:nodoc:
       #   <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
       #   ActiveSupport::Multibyte::Unicode.default_normalization_form
       def normalize(form = nil)
-        chars(Unicode.normalize(@wrapped_string, form))
+        form ||= Unicode.default_normalization_form
+
+        # See https://www.unicode.org/reports/tr15, Table 1
+        if alias_form = Unicode::NORMALIZATION_FORM_ALIASES[form]
+          ActiveSupport::Deprecation.warn(<<-MSG.squish)
+            ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
+            removed from Rails 6.1. Use #unicode_normalize(:#{alias_form}) instead.
+          MSG
+
+          send(:unicode_normalize, alias_form)
+        else
+          ActiveSupport::Deprecation.warn(<<-MSG.squish)
+            ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
+            removed from Rails 6.1. Use #unicode_normalize instead.
+          MSG
+
+          raise ArgumentError, "#{form} is not a valid normalization variant", caller
+        end
       end
 
       # Performs canonical decomposition on all the characters.
@@ -190,7 +183,7 @@ module ActiveSupport #:nodoc:
       #   'क्षि'.mb_chars.length   # => 4
       #   'क्षि'.mb_chars.grapheme_length # => 3
       def grapheme_length
-        Unicode.unpack_graphemes(@wrapped_string).length
+        @wrapped_string.scan(/\X/).length
       end
 
       # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent
@@ -206,7 +199,7 @@ module ActiveSupport #:nodoc:
         to_s.as_json(options)
       end
 
-      %w(capitalize downcase reverse tidy_bytes upcase).each do |method|
+      %w(reverse tidy_bytes).each do |method|
         define_method("#{method}!") do |*args|
           @wrapped_string = send(method, *args).to_s
           self
@@ -215,18 +208,6 @@ module ActiveSupport #:nodoc:
 
       private
 
-        def translate_offset(byte_offset)
-          return nil if byte_offset.nil?
-          return 0   if @wrapped_string == ""
-
-          begin
-            @wrapped_string.byteslice(0...byte_offset).unpack("U*").length
-          rescue ArgumentError
-            byte_offset -= 1
-            retry
-          end
-        end
-
         def chars(string)
           self.class.new(string)
         end

data/lib/active_support/multibyte/unicode.rb

@@ -6,12 +6,19 @@ module ActiveSupport
       extend self
 
       # A list of all available normalization forms.
-      # See http://www.unicode.org/reports/tr15/tr15-29.html for more
+      # See https://www.unicode.org/reports/tr15/tr15-29.html for more
       # information about normalization.
       NORMALIZATION_FORMS = [:c, :kc, :d, :kd]
 
+      NORMALIZATION_FORM_ALIASES = { # :nodoc:
+        c: :nfc,
+        d: :nfd,
+        kc: :nfkc,
+        kd: :nfkd
+      }
+
       # The Unicode version that is supported by the implementation
-      UNICODE_VERSION = "9.0.0"
+      UNICODE_VERSION = RbConfig::CONFIG["UNICODE_VERSION"]
 
       # The default normalization used for operations that require
       # normalization. It can be set to any of the normalizations
@@ -21,199 +28,44 @@ module ActiveSupport
       attr_accessor :default_normalization_form
       @default_normalization_form = :kc
 
-      # Hangul character boundaries and properties
-      HANGUL_SBASE = 0xAC00
-      HANGUL_LBASE = 0x1100
-      HANGUL_VBASE = 0x1161
-      HANGUL_TBASE = 0x11A7
-      HANGUL_LCOUNT = 19
-      HANGUL_VCOUNT = 21
-      HANGUL_TCOUNT = 28
-      HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
-      HANGUL_SCOUNT = 11172
-      HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
-
-      # Detect whether the codepoint is in a certain character class. Returns
-      # +true+ when it's in the specified character class and +false+ otherwise.
-      # Valid character classes are: <tt>:cr</tt>, <tt>:lf</tt>, <tt>:l</tt>,
-      # <tt>:v</tt>, <tt>:lv</tt>, <tt>:lvt</tt> and <tt>:t</tt>.
-      #
-      # Primarily used by the grapheme cluster support.
-      def in_char_class?(codepoint, classes)
-        classes.detect { |c| database.boundary[c] === codepoint } ? true : false
-      end
-
       # Unpack the string at grapheme boundaries. Returns a list of character
       # lists.
       #
       #   Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
       #   Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]
       def unpack_graphemes(string)
-        codepoints = string.codepoints.to_a
-        unpacked = []
-        pos = 0
-        marker = 0
-        eoc = codepoints.length
-        while (pos < eoc)
-          pos += 1
-          previous = codepoints[pos - 1]
-          current = codepoints[pos]
-
-          # See http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
-          should_break =
-            if pos == eoc
-              true
-            # GB3. CR X LF
-            elsif previous == database.boundary[:cr] && current == database.boundary[:lf]
-              false
-            # GB4. (Control|CR|LF) ÷
-            elsif previous && in_char_class?(previous, [:control, :cr, :lf])
-              true
-            # GB5. ÷ (Control|CR|LF)
-            elsif in_char_class?(current, [:control, :cr, :lf])
-              true
-            # GB6. L X (L|V|LV|LVT)
-            elsif database.boundary[:l] === previous && in_char_class?(current, [:l, :v, :lv, :lvt])
-              false
-            # GB7. (LV|V) X (V|T)
-            elsif in_char_class?(previous, [:lv, :v]) && in_char_class?(current, [:v, :t])
-              false
-            # GB8. (LVT|T) X (T)
-            elsif in_char_class?(previous, [:lvt, :t]) && database.boundary[:t] === current
-              false
-            # GB9. X (Extend | ZWJ)
-            elsif in_char_class?(current, [:extend, :zwj])
-              false
-            # GB9a. X SpacingMark
-            elsif database.boundary[:spacingmark] === current
-              false
-            # GB9b. Prepend X
-            elsif database.boundary[:prepend] === previous
-              false
-            # GB10. (E_Base | EBG) Extend* X E_Modifier
-            elsif (marker...pos).any? { |i| in_char_class?(codepoints[i], [:e_base, :e_base_gaz]) && codepoints[i + 1...pos].all? { |c| database.boundary[:extend] === c } } && database.boundary[:e_modifier] === current
-              false
-            # GB11. ZWJ X (Glue_After_Zwj | EBG)
-            elsif database.boundary[:zwj] === previous && in_char_class?(current, [:glue_after_zwj, :e_base_gaz])
-              false
-            # GB12. ^ (RI RI)* RI X RI
-            # GB13. [^RI] (RI RI)* RI X RI
-            elsif codepoints[marker..pos].all? { |c| database.boundary[:regional_indicator] === c } && codepoints[marker..pos].count { |c| database.boundary[:regional_indicator] === c }.even?
-              false
-            # GB999. Any ÷ Any
-            else
-              true
-            end
+        ActiveSupport::Deprecation.warn(<<-MSG.squish)
+          ActiveSupport::Multibyte::Unicode#unpack_graphemes is deprecated and will be
+          removed from Rails 6.1. Use string.scan(/\X/).map(&:codepoints) instead.
+        MSG
 
-          if should_break
-            unpacked << codepoints[marker..pos - 1]
-            marker = pos
-          end
-        end
-        unpacked
+        string.scan(/\X/).map(&:codepoints)
       end
 
       # Reverse operation of unpack_graphemes.
       #
       #   Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'
       def pack_graphemes(unpacked)
-        unpacked.flatten.pack("U*")
-      end
+        ActiveSupport::Deprecation.warn(<<-MSG.squish)
+          ActiveSupport::Multibyte::Unicode#pack_graphemes is deprecated and will be
+          removed from Rails 6.1. Use array.flatten.pack("U*") instead.
+        MSG
 
-      # Re-order codepoints so the string becomes canonical.
-      def reorder_characters(codepoints)
-        length = codepoints.length - 1
-        pos = 0
-        while pos < length do
-          cp1, cp2 = database.codepoints[codepoints[pos]], database.codepoints[codepoints[pos + 1]]
-          if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
-            codepoints[pos..pos + 1] = cp2.code, cp1.code
-            pos += (pos > 0 ? -1 : 1)
-          else
-            pos += 1
-          end
-        end
-        codepoints
+        unpacked.flatten.pack("U*")
       end
 
       # Decompose composed characters to the decomposed form.
       def decompose(type, codepoints)
-        codepoints.inject([]) do |decomposed, cp|
-          # if it's a hangul syllable starter character
-          if HANGUL_SBASE <= cp && cp < HANGUL_SLAST
-            sindex = cp - HANGUL_SBASE
-            ncp = [] # new codepoints
-            ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
-            ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
-            tindex = sindex % HANGUL_TCOUNT
-            ncp << (HANGUL_TBASE + tindex) unless tindex == 0
-            decomposed.concat ncp
-          # if the codepoint is decomposable in with the current decomposition type
-          elsif (ncp = database.codepoints[cp].decomp_mapping) && (!database.codepoints[cp].decomp_type || type == :compatibility)
-            decomposed.concat decompose(type, ncp.dup)
-          else
-            decomposed << cp
-          end
+        if type == :compatibility
+          codepoints.pack("U*").unicode_normalize(:nfkd).codepoints
+        else
+          codepoints.pack("U*").unicode_normalize(:nfd).codepoints
         end
       end
 
       # Compose decomposed characters to the composed form.
       def compose(codepoints)
-        pos = 0
-        eoa = codepoints.length - 1
-        starter_pos = 0
-        starter_char = codepoints[0]
-        previous_combining_class = -1
-        while pos < eoa
-          pos += 1
-          lindex = starter_char - HANGUL_LBASE
-          # -- Hangul
-          if 0 <= lindex && lindex < HANGUL_LCOUNT
-            vindex = codepoints[starter_pos + 1] - HANGUL_VBASE rescue vindex = -1
-            if 0 <= vindex && vindex < HANGUL_VCOUNT
-              tindex = codepoints[starter_pos + 2] - HANGUL_TBASE rescue tindex = -1
-              if 0 <= tindex && tindex < HANGUL_TCOUNT
-                j = starter_pos + 2
-                eoa -= 2
-              else
-                tindex = 0
-                j = starter_pos + 1
-                eoa -= 1
-              end
-              codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
-            end
-            starter_pos += 1
-            starter_char = codepoints[starter_pos]
-          # -- Other characters
-          else
-            current_char = codepoints[pos]
-            current = database.codepoints[current_char]
-            if current.combining_class > previous_combining_class
-              if ref = database.composition_map[starter_char]
-                composition = ref[current_char]
-              else
-                composition = nil
-              end
-              unless composition.nil?
-                codepoints[starter_pos] = composition
-                starter_char = composition
-                codepoints.delete_at pos
-                eoa -= 1
-                pos -= 1
-                previous_combining_class = -1
-              else
-                previous_combining_class = current.combining_class
-              end
-            else
-              previous_combining_class = current.combining_class
-            end
-            if current.combining_class == 0
-              starter_pos = pos
-              starter_char = codepoints[pos]
-            end
-          end
-        end
-        codepoints
+        codepoints.pack("U*").unicode_normalize(:nfc).codepoints
       end
 
       # Rubinius' String#scrub, however, doesn't support ASCII-incompatible chars.
@@ -265,130 +117,41 @@ module ActiveSupport
       #   Default is ActiveSupport::Multibyte::Unicode.default_normalization_form.
       def normalize(string, form = nil)
         form ||= @default_normalization_form
-        # See http://www.unicode.org/reports/tr15, Table 1
-        codepoints = string.codepoints.to_a
-        case form
-        when :d
-          reorder_characters(decompose(:canonical, codepoints))
-        when :c
-          compose(reorder_characters(decompose(:canonical, codepoints)))
-        when :kd
-          reorder_characters(decompose(:compatibility, codepoints))
-        when :kc
-          compose(reorder_characters(decompose(:compatibility, codepoints)))
-        else
-          raise ArgumentError, "#{form} is not a valid normalization variant", caller
-        end.pack("U*".freeze)
-      end
-
-      def downcase(string)
-        apply_mapping string, :lowercase_mapping
-      end
 
-      def upcase(string)
-        apply_mapping string, :uppercase_mapping
-      end
+        # See https://www.unicode.org/reports/tr15, Table 1
+        if alias_form = NORMALIZATION_FORM_ALIASES[form]
+          ActiveSupport::Deprecation.warn(<<-MSG.squish)
+            ActiveSupport::Multibyte::Unicode#normalize is deprecated and will be
+            removed from Rails 6.1. Use String#unicode_normalize(:#{alias_form}) instead.
+          MSG
 
-      def swapcase(string)
-        apply_mapping string, :swapcase_mapping
-      end
-
-      # Holds data about a codepoint in the Unicode database.
-      class Codepoint
-        attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping
-
-        # Initializing Codepoint object with default values
-        def initialize
-          @combining_class = 0
-          @uppercase_mapping = 0
-          @lowercase_mapping = 0
-        end
+          string.unicode_normalize(alias_form)
+        else
+          ActiveSupport::Deprecation.warn(<<-MSG.squish)
+            ActiveSupport::Multibyte::Unicode#normalize is deprecated and will be
+            removed from Rails 6.1. Use String#unicode_normalize instead.
+          MSG
 
-        def swapcase_mapping
-          uppercase_mapping > 0 ? uppercase_mapping : lowercase_mapping
+          raise ArgumentError, "#{form} is not a valid normalization variant", caller
         end
       end
 
-      # Holds static data from the Unicode database.
-      class UnicodeDatabase
-        ATTRIBUTES = :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
-
-        attr_writer(*ATTRIBUTES)
+      %w(downcase upcase swapcase).each do |method|
+        define_method(method) do |string|
+          ActiveSupport::Deprecation.warn(<<-MSG.squish)
+          ActiveSupport::Multibyte::Unicode##{method} is deprecated and
+          will be removed from Rails 6.1. Use String methods directly.
+          MSG
 
-        def initialize
-          @codepoints = Hash.new(Codepoint.new)
-          @composition_exclusion = []
-          @composition_map = {}
-          @boundary = {}
-          @cp1252 = {}
-        end
-
-        # Lazy load the Unicode database so it's only loaded when it's actually used
-        ATTRIBUTES.each do |attr_name|
-          class_eval(<<-EOS, __FILE__, __LINE__ + 1)
-            def #{attr_name}     # def codepoints
-              load               #   load
-              @#{attr_name}      #   @codepoints
-            end                  # end
-          EOS
-        end
-
-        # Loads the Unicode database and returns all the internal objects of
-        # UnicodeDatabase.
-        def load
-          begin
-            @codepoints, @composition_exclusion, @composition_map, @boundary, @cp1252 = File.open(self.class.filename, "rb") { |f| Marshal.load f.read }
-          rescue => e
-            raise IOError.new("Couldn't load the Unicode tables for UTF8Handler (#{e.message}), ActiveSupport::Multibyte is unusable")
-          end
-
-          # Redefine the === method so we can write shorter rules for grapheme cluster breaks
-          @boundary.each_key do |k|
-            @boundary[k].instance_eval do
-              def ===(other)
-                detect { |i| i === other } ? true : false
-              end
-            end if @boundary[k].kind_of?(Array)
-          end
-
-          # define attr_reader methods for the instance variables
-          class << self
-            attr_reader(*ATTRIBUTES)
-          end
-        end
-
-        # Returns the directory in which the data files are stored.
-        def self.dirname
-          File.expand_path("../values", __dir__)
-        end
-
-        # Returns the filename for the data file for this version.
-        def self.filename
-          File.expand_path File.join(dirname, "unicode_tables.dat")
+          string.send(method)
         end
       end
 
       private
 
-        def apply_mapping(string, mapping)
-          database.codepoints
-          string.each_codepoint.map do |codepoint|
-            cp = database.codepoints[codepoint]
-            if cp && (ncp = cp.send(mapping)) && ncp > 0
-              ncp
-            else
-              codepoint
-            end
-          end.pack("U*")
-        end
-
         def recode_windows1252_chars(string)
           string.encode(Encoding::UTF_8, Encoding::Windows_1252, invalid: :replace, undef: :replace)
         end
-
-        def database
-          @database ||= UnicodeDatabase.new
-        end
     end
   end
 end