character_set 1.3.0-java → 1.6.0-java

data/lib/character_set/predefined_sets/assigned.cps

@@ -21,7 +21,7 @@
 85E,85E
 860,86A
 8A0,8B4
-8B6,8BD
+8B6,8C7
 8D3,983
 985,98C
 98F,990
@@ -76,7 +76,7 @@ B35,B39
 B3C,B44
 B47,B48
 B4B,B4D
-B56,B57
+B55,B57
 B5C,B5D
 B5F,B63
 B66,B77
@@ -120,15 +120,14 @@ CDE,CDE
 CE0,CE3
 CE6,CEF
 CF1,CF2
-D00,D03
-D05,D0C
+D00,D0C
 D0E,D10
 D12,D44
 D46,D48
 D4A,D4F
 D54,D63
 D66,D7F
-D82,D83
+D81,D83
 D85,D96
 D9A,DB1
 DB3,DBB
@@ -214,7 +213,7 @@ FCE,FDA
 1A7F,1A89
 1A90,1A99
 1AA0,1AAD
-1AB0,1ABE
+1AB0,1AC0
 1B00,1B4B
 1B50,1B7C
 1B80,1BF3
@@ -252,7 +251,7 @@ FCE,FDA
 2440,244A
 2460,2B73
 2B76,2B95
-2B98,2C2E
+2B97,2C2E
 2C30,2C5E
 2C60,2CF3
 2CF9,2D25
@@ -269,7 +268,7 @@ FCE,FDA
 2DC8,2DCE
 2DD0,2DD6
 2DD8,2DDE
-2DE0,2E4F
+2DE0,2E52
 2E80,2E99
 2E9B,2EF3
 2F00,2FD5
@@ -279,18 +278,16 @@ FCE,FDA
 3099,30FF
 3105,312F
 3131,318E
-3190,31BA
-31C0,31E3
+3190,31E3
 31F0,321E
-3220,4DB5
-4DC0,9FEF
+3220,9FFC
 A000,A48C
 A490,A4C6
 A4D0,A62B
 A640,A6F7
 A700,A7BF
-A7C2,A7C6
-A7F7,A82B
+A7C2,A7CA
+A7F5,A82C
 A830,A839
 A840,A877
 A880,A8C5
@@ -310,7 +307,7 @@ AB09,AB0E
 AB11,AB16
 AB20,AB26
 AB28,AB2E
-AB30,AB67
+AB30,AB6B
 AB70,ABED
 ABF0,ABF9
 AC00,D7A3
@@ -355,7 +352,7 @@ FFF9,FFFD
 10100,10102
 10107,10133
 10137,1018E
-10190,1019B
+10190,1019C
 101A0,101A0
 101D0,101FD
 10280,1029C
@@ -415,8 +412,12 @@ FFF9,FFFD
 10CFA,10D27
 10D30,10D39
 10E60,10E7E
+10E80,10EA9
+10EAB,10EAD
+10EB0,10EB1
 10F00,10F27
 10F30,10F59
+10FB0,10FCB
 10FE0,10FF6
 11000,1104D
 11052,1106F
@@ -425,10 +426,9 @@ FFF9,FFFD
 110D0,110E8
 110F0,110F9
 11100,11134
-11136,11146
+11136,11147
 11150,11176
-11180,111CD
-111D0,111DF
+11180,111DF
 111E1,111F4
 11200,11211
 11213,1123E
@@ -454,9 +454,8 @@ FFF9,FFFD
 1135D,11363
 11366,1136C
 11370,11374
-11400,11459
-1145B,1145B
-1145D,1145F
+11400,1145B
+1145D,11461
 11480,114C7
 114D0,114D9
 11580,115B5
@@ -471,7 +470,14 @@ FFF9,FFFD
 11730,1173F
 11800,1183B
 118A0,118F2
-118FF,118FF
+118FF,11906
+11909,11909
+1190C,11913
+11915,11916
+11918,11935
+11937,11938
+1193B,11946
+11950,11959
 119A0,119A7
 119AA,119D7
 119DA,119E4
@@ -499,6 +505,7 @@ FFF9,FFFD
 11D93,11D98
 11DA0,11DA9
 11EE0,11EF8
+11FB0,11FB0
 11FC0,11FF1
 11FFF,12399
 12400,1246E
@@ -522,9 +529,11 @@ FFF9,FFFD
 16F00,16F4A
 16F4F,16F87
 16F8F,16F9F
-16FE0,16FE3
+16FE0,16FE4
+16FF0,16FF1
 17000,187F7
-18800,18AF2
+18800,18CD5
+18D00,18D08
 1B000,1B11E
 1B150,1B152
 1B164,1B167
@@ -622,17 +631,15 @@ FFF9,FFFD
 1F0B1,1F0BF
 1F0C1,1F0CF
 1F0D1,1F0F5
-1F100,1F10C
-1F110,1F16C
-1F170,1F1AC
+1F100,1F1AD
 1F1E6,1F202
 1F210,1F23B
 1F240,1F248
 1F250,1F251
 1F260,1F265
-1F300,1F6D5
+1F300,1F6D7
 1F6E0,1F6EC
-1F6F0,1F6FA
+1F6F0,1F6FC
 1F700,1F773
 1F780,1F7D8
 1F7E0,1F7EB
@@ -641,24 +648,28 @@ FFF9,FFFD
 1F850,1F859
 1F860,1F887
 1F890,1F8AD
-1F900,1F90B
-1F90D,1F971
-1F973,1F976
-1F97A,1F9A2
-1F9A5,1F9AA
-1F9AE,1F9CA
+1F8B0,1F8B1
+1F900,1F978
+1F97A,1F9CB
 1F9CD,1FA53
 1FA60,1FA6D
-1FA70,1FA73
+1FA70,1FA74
 1FA78,1FA7A
-1FA80,1FA82
-1FA90,1FA95
-20000,2A6D6
+1FA80,1FA86
+1FA90,1FAA8
+1FAB0,1FAB6
+1FAC0,1FAC2
+1FAD0,1FAD6
+1FB00,1FB92
+1FB94,1FBCA
+1FBF0,1FBF9
+20000,2A6DD
 2A700,2B734
 2B740,2B81D
 2B820,2CEA1
 2CEB0,2EBE0
 2F800,2FA1D
+30000,3134A
 E0001,E0001
 E0020,E007F
 E0100,E01EF

data/lib/character_set/predefined_sets/emoji.cps

@@ -44,6 +44,7 @@ AE,AE
 2699,2699
 269B,269C
 26A0,26A1
+26A7,26A7
 26AA,26AB
 26B0,26B1
 26BD,26BE
@@ -130,22 +131,22 @@ AE,AE
 1F5FA,1F64F
 1F680,1F6C5
 1F6CB,1F6D2
-1F6D5,1F6D5
+1F6D5,1F6D7
 1F6E0,1F6E5
 1F6E9,1F6E9
 1F6EB,1F6EC
 1F6F0,1F6F0
-1F6F3,1F6FA
+1F6F3,1F6FC
 1F7E0,1F7EB
-1F90D,1F93A
+1F90C,1F93A
 1F93C,1F945
-1F947,1F971
-1F973,1F976
-1F97A,1F9A2
-1F9A5,1F9AA
-1F9AE,1F9CA
+1F947,1F978
+1F97A,1F9CB
 1F9CD,1F9FF
-1FA70,1FA73
+1FA70,1FA74
 1FA78,1FA7A
-1FA80,1FA82
-1FA90,1FA95
+1FA80,1FA86
+1FA90,1FAA8
+1FAB0,1FAB6
+1FAC0,1FAC2
+1FAD0,1FAD6

data/lib/character_set/predefined_sets.rb

@@ -22,6 +22,17 @@ class CharacterSet
     alias valid                    unicode
 
     def build_from_cps_file(path)
+      if defined?(Ractor) && Ractor.current != Ractor.main
+        raise <<-EOS.gsub(/^ */, '')
+          CharacterSet's predefined sets are lazy-loaded.
+          Pre-load them to use them in Ractors. E.g.:
+
+          CharacterSet.ascii # pre-load
+          Ractor.new { CharacterSet.ascii.size }.take # => 128
+          Ractor.new { 'abc'.keep_character_set(:ascii) }.take # => 'abc'
+        EOS
+      end
+
       File.readlines(path).inject(new) do |set, line|
         range_start, range_end = line.split(',')
         set.merge((range_start.to_i(16))..(range_end.to_i(16)))

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -6,9 +6,9 @@ class CharacterSet
           new(Array(ranges).flat_map(&:to_a))
         end
 
-        def of(string)
-          raise ArgumentError, 'pass a String' unless string.is_a?(String)
-          new(string.codepoints)
+        def of_string(str)
+          raise ArgumentError, 'pass a String' unless str.respond_to?(:codepoints)
+          str.codepoints.each_with_object(new) { |cp, set| set << cp }
         end
       end
 
@@ -31,7 +31,7 @@ class CharacterSet
       end
 
       def ranges
-        CharacterSet.require_optional_dependency('range_compressor')
+        CharacterSet.require_optional_dependency('range_compressor', __method__)
         RangeCompressor.compress(self)
       end
 
@@ -121,10 +121,9 @@ class CharacterSet
       end
 
       def make_new_str(original, &block)
-        new_string = str!(original)
+        str!(original)
           .each_codepoint
           .each_with_object(''.encode(original.encoding), &block)
-        original.tainted? ? new_string.taint : new_string
       end
     end
   end

data/lib/character_set/ruby_fallback/set_methods.rb

@@ -1,7 +1,9 @@
 class CharacterSet
   module RubyFallback
     module SetMethods
-      Enumerable.instance_methods.concat(%w[empty? length size]).each do |mthd|
+      (Enumerable.instance_methods -
+        %i[include? member? to_a] +
+        %i[empty? length size]).each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(*args, &block)
             @__set.#{mthd}(*args, &block)
@@ -9,7 +11,7 @@ class CharacterSet
         RUBY
       end
 
-      %w[< <= > >= disjoint? intersect? proper_subset? proper_superset?
+      %i[< <= > >= disjoint? intersect? proper_subset? proper_superset?
          subset? superset?].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(enum, &block)
@@ -21,8 +23,8 @@ class CharacterSet
         RUBY
       end
 
-      %w[<< === add add? clear collect! delete delete? delete_if
-         each filter! hash include? map! member? keep_if reject!
+      %i[<< add add? clear collect! delete delete? delete_if
+         each filter! map! keep_if reject!
          select! subtract].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(*args, &block)
@@ -32,22 +34,22 @@ class CharacterSet
         RUBY
       end
 
-      %w[& + - ^ | difference intersection union].each do |mthd|
+      # revert if https://github.com/knu/sorted_set/issues/2 is resolved
+      %i[=== include? member?].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{mthd}(enum, &block)
-            if enum.respond_to?(:map)
-              enum = enum.map { |el| el.is_a?(String) ? el.ord : el }
-            end
-            self.class.new(@__set.#{mthd}(enum, &block).to_a)
+          def #{mthd}(*args, &block)
+            !!@__set.#{mthd}(*args, &block)
           end
         RUBY
       end
 
-      %w[taint untaint].each do |mthd|
+      %i[& + - ^ | difference intersection union].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{mthd}
-            @__set.#{mthd}
-            super
+          def #{mthd}(enum, &block)
+            if enum.respond_to?(:map)
+              enum = enum.map { |el| el.is_a?(String) ? el.ord : el }
+            end
+            self.class.new(@__set.#{mthd}(enum, &block).to_a)
           end
         RUBY
       end
@@ -81,7 +83,13 @@ class CharacterSet
 
       def eql?(other)
         return false unless other.is_a?(self.class)
-        @__set.eql?(other.instance_variable_get(:@__set))
+        # revert if https://github.com/knu/sorted_set/issues/3 is resolved
+        hash == other.hash
+      end
+
+      # revert if https://github.com/knu/sorted_set/issues/3 is resolved
+      def hash
+        @__set.to_a.hash
       end
 
       def initialize_dup(orig)

data/lib/character_set/ruby_fallback.rb

@@ -1,4 +1,8 @@
-require 'set'
+if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+  require 'sorted_set'
+else
+  require 'set'
+end
 require 'character_set/ruby_fallback/set_methods'
 require 'character_set/ruby_fallback/character_set_methods'
 

data/lib/character_set/set_method_adapters.rb

@@ -22,13 +22,14 @@ class CharacterSet
 
     # Allow some methods to take an Enum just as well as another CharacterSet.
     # Tested by ruby-spec.
-    %w[& + - ^ | difference intersection subtract union].each do |method|
+    %w[& + - ^ | difference disjoint? intersect? intersection
+       subtract union].each do |method|
       class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{method}(arg)
           if arg.is_a?(CharacterSet)
-            super
+            super(arg)
           elsif arg.respond_to?(:each)
-            super(CharacterSet.new(arg.to_a))
+            super(self.class.new(arg.to_a))
           else
             raise ArgumentError, 'pass an enumerable'
           end

data/lib/character_set/shared_methods.rb

@@ -15,6 +15,12 @@ class CharacterSet
             new(Array(args))
           end
 
+          def of(*args)
+            args.map do |arg|
+              arg.is_a?(Regexp) ? of_regexp(arg) : of_string(arg)
+            end.reduce(:merge) || new
+          end
+
           def parse(string)
             codepoints = Parser.codepoints_from_bracket_expression(string)
             result = new(codepoints)
@@ -22,33 +28,29 @@ class CharacterSet
           end
 
           def of_property(property_name)
-            require_optional_dependency('regexp_property_values')
+            require_optional_dependency('regexp_property_values', __method__)
 
             property = RegexpPropertyValues[property_name.to_s]
             from_ranges(*property.matched_ranges)
           end
 
           def of_regexp(regexp)
-            require_optional_dependency('regexp_parser')
+            require_optional_dependency('regexp_parser', __method__)
 
             root = ::Regexp::Parser.parse(regexp)
             of_expression(root)
           end
 
           def of_expression(expression)
-            ExpressionConverter.convert(expression)
+            ExpressionConverter.convert(expression, self)
           end
 
-          def require_optional_dependency(name)
+          def require_optional_dependency(name, method)
             required_optional_dependencies[name] ||= begin
               require name
               true
             rescue ::LoadError
-              entry_point = caller_locations.reverse.find do |loc|
-                loc.absolute_path.to_s.include?('/lib/character_set')
-              end
-              method = entry_point && entry_point.label
-              raise LoadError, 'You must the install the optional dependency '\
+              raise LoadError, 'You must install the optional dependency '\
                                "'\#{name}' to use the method `\#{method}'."
             end
           end
@@ -86,10 +88,22 @@ class CharacterSet
           Writer.write(ranges, opts, &block)
         end
 
+        def to_s_with_surrogate_ranges
+          Writer.write_surrogate_ranges(bmp_part.ranges, astral_part.ranges)
+        end
+
         def to_s_with_surrogate_alternation
           Writer.write_surrogate_alternation(bmp_part.ranges, astral_part.ranges)
         end
 
+        def secure_token(length = 32)
+          CharacterSet.require_optional_dependency('securerandom', __method__)
+          cps = to_a
+          len = cps.count
+          1.upto(length).map { cps[SecureRandom.random_number(len)] }.pack('U*')
+        end
+        alias random_token secure_token
+
         def inspect
           len = length
           "#<#{klass.name}: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"
@@ -151,7 +165,7 @@ class CharacterSet
         end
 
         def divide(&func)
-          require 'set'
+          CharacterSet.require_optional_dependency('set', __method__)
           Set.new(to_a).divide(&func)
         end
       RUBY

data/lib/character_set/version.rb

@@ -1,3 +1,3 @@
 class CharacterSet
-  VERSION = '1.3.0'
+  VERSION = '1.6.0'
 end

data/lib/character_set/writer.rb

@@ -1,37 +1,108 @@
 class CharacterSet
   module Writer
-    module_function
-
-    def write(codepoint_ranges, opts = {}, &block)
-      content = codepoint_ranges.map do |range|
-        if range.size > 2 && opts[:abbreviate] != false
-          range.minmax.map { |cp| Character.new(cp).escape(opts, &block) }.join('-')
-        else
-          range.map { |cp| Character.new(cp).escape(opts, &block) }.join
+    class << self
+      def write(codepoint_ranges, opts = {}, &block)
+        content = codepoint_ranges.map do |range|
+          if range.size > 2 && opts[:abbreviate] != false
+            bounds = [range.min, range.max]
+            bounds.map { |cp| write_codepoint(cp, opts, &block) }.join('-')
+          else
+            range.map { |cp| write_codepoint(cp, opts, &block) }.join
+          end
+        end.join
+        opts[:in_brackets] ? "[#{content}]" : content
+      end
+
+      def write_codepoint(codepoint, opts = {}, &block)
+        Character.new(codepoint).escape(opts, &block)
+      end
+
+      def write_surrogate_ranges(bmp_ranges, astral_ranges)
+        astral_branches = surrogate_range_expressions(astral_ranges)
+        bmp_set_with_alternatives(bmp_ranges, astral_branches)
+      end
+
+      def write_surrogate_alternation(bmp_ranges, astral_ranges)
+        astral_branches = surrogate_pairs(astral_ranges)
+        bmp_set_with_alternatives(bmp_ranges, astral_branches)
+      end
+
+      private
+
+      def surrogate_range_expressions(astral_ranges)
+        compressed_surrogate_range_pairs(astral_ranges).map do |hi_ranges, lo_ranges|
+          [hi_ranges, lo_ranges].map do |ranges|
+            use_brackets = ranges.size > 1 || ranges.first.size > 1
+            write(ranges, format: :js, in_brackets: use_brackets)
+          end.join
         end
-      end.join
-      opts[:in_brackets] ? "[#{content}]" : content
-    end
+      end
+
+      def compressed_surrogate_range_pairs(astral_ranges)
+        halves = astral_ranges.flat_map { |range| surrogate_half_ranges(range) }
 
-    def write_surrogate_alternation(bmp_ranges, astral_ranges)
-      bmp_set = write(bmp_ranges, format: :js, in_brackets: true)
-      if astral_ranges.empty?
-        bmp_set
-      else
-        surrogate_pairs = surrogate_pairs(astral_ranges)
-        "(?:#{((bmp_ranges.any? ? [bmp_set] : []) + surrogate_pairs) * '|'})"
+        # compress high surrogate codepoint ranges with common low range half
+        with_common_lo = halves.group_by(&:last).map do |lo_range, pairs|
+          hi_ranges = pairs.map(&:first)
+          compressed_hi_ranges = hi_ranges.each_with_object([]) do |range, arr|
+            prev = arr.last
+            if prev.nil? || prev.max + 1 < range.min # first or gap
+              arr << range
+            else # continuous codepoints, expand previous range
+              arr[-1] = (prev.min)..(range.max)
+            end
+          end
+          [compressed_hi_ranges, lo_range]
+        end
+
+        # compress low surrogate codepoint ranges with common high ranges
+        with_common_lo.each_with_object({}) do |(hi_ranges, lo_range), hash|
+          (hash[hi_ranges] ||= []) << lo_range
+        end
       end
-    end
 
-    def surrogate_pairs(astral_ranges)
-      astral_ranges.flat_map { |range| range.map { |cp| surrogate_pair(cp) } }
-    end
+      def surrogate_half_ranges(astral_range)
+        hi_min, lo_min = surrogate_pair_codepoints(astral_range.min)
+        hi_max, lo_max = surrogate_pair_codepoints(astral_range.max)
+        hi_count = 1 + hi_max - hi_min
+        return [[hi_min..hi_min, lo_min..lo_max]] if hi_count == 1
+
+        ranges = []
+
+        # first high surrogate might be partially covered (if lo_min > 0xDC00)
+        ranges << [hi_min..hi_min, lo_min..0xDFFF]
+
+        # any high surrogates in between are fully covered
+        ranges << [(hi_min + 1)..(hi_max - 1), 0xDC00..0xDFFF] if hi_count > 2
 
-    def surrogate_pair(astral_codepoint)
-      base = astral_codepoint - 0x10000
-      high = ((base / 1024).floor + 0xD800).to_s(16)
-      low  = (base % 1024 + 0xDC00).to_s(16)
-      "\\u#{high}\\u#{low}"
+        # last high surrogate might be partially covered (if lo_max < 0xDFFF)
+        ranges << [hi_max..hi_max, 0xDC00..lo_max]
+
+        ranges
+      end
+
+      def surrogate_pair_codepoints(astral_codepoint)
+        base = astral_codepoint - 0x10000
+        high = base / 1024 + 0xD800
+        low  = base % 1024 + 0xDC00
+        [high, low]
+      end
+
+      def bmp_set_with_alternatives(bmp_ranges, alternatives)
+        bmp_set = write(bmp_ranges, format: :js, in_brackets: true)
+        return bmp_set if alternatives.empty? && bmp_ranges.any?
+
+        "(?:#{((bmp_ranges.any? ? [bmp_set] : []) + alternatives).join('|')})"
+      end
+
+      def surrogate_pairs(astral_ranges)
+        astral_ranges.flat_map { |range| range.map { |cp| surrogate_pair(cp) } }
+      end
+
+      def surrogate_pair(astral_codepoint)
+        surrogate_pair_codepoints(astral_codepoint)
+          .map { |half| write_codepoint(half, format: :js) }.join
+      end
     end
   end
 end