character_set 1.4.0 → 1.8.0

data/lib/character_set/parser.rb

@@ -4,11 +4,15 @@ class CharacterSet
 
     def codepoints_from_enumerable(object)
       raise ArgumentError, 'pass an Enumerable' unless object.respond_to?(:each)
+
       # Use #each to check first element (only this works for all Enumerables)
-      object.each do |e|
-        return object            if e.is_a?(Integer) && e >= 0 && e < 0x110000
-        return object.map(&:ord) if e.is_a?(String)  && e.length == 1
-        raise ArgumentError, "#{e.inspect} is not valid as a codepoint"
+      object.each do |el| # rubocop:disable Lint/UnreachableLoop
+        if el.is_a?(Integer) && el >= 0 && el < 0x110000
+          return object
+        elsif el.is_a?(String) && el.length == 1
+          return object.to_a.join.encode('utf-8').codepoints
+        end
+        raise ArgumentError, "#{el.inspect} is not valid as a codepoint"
       end
     end
 

data/lib/character_set/predefined_sets/assigned.cps

@@ -10,8 +10,7 @@
 591,5C7
 5D0,5EA
 5EF,5F4
-600,61C
-61E,70D
+600,70D
 70F,74A
 74D,7B1
 7C0,7FA
@@ -20,9 +19,9 @@
 840,85B
 85E,85E
 860,86A
-8A0,8B4
-8B6,8BD
-8D3,983
+870,88E
+890,891
+898,983
 985,98C
 98F,990
 993,9A8
@@ -76,7 +75,7 @@ B35,B39
 B3C,B44
 B47,B48
 B4B,B4D
-B56,B57
+B55,B57
 B5C,B5D
 B5F,B63
 B66,B77
@@ -100,11 +99,12 @@ C00,C0C
 C0E,C10
 C12,C28
 C2A,C39
-C3D,C44
+C3C,C44
 C46,C48
 C4A,C4D
 C55,C56
 C58,C5A
+C5D,C5D
 C60,C63
 C66,C6F
 C77,C8C
@@ -116,19 +116,18 @@ CBC,CC4
 CC6,CC8
 CCA,CCD
 CD5,CD6
-CDE,CDE
+CDD,CDE
 CE0,CE3
 CE6,CEF
 CF1,CF2
-D00,D03
-D05,D0C
+D00,D0C
 D0E,D10
 D12,D44
 D46,D48
 D4A,D4F
 D54,D63
 D66,D7F
-D82,D83
+D81,D83
 D85,D96
 D9A,DB1
 DB3,DBB
@@ -184,9 +183,8 @@ FCE,FDA
 13F8,13FD
 1400,169C
 16A0,16F8
-1700,170C
-170E,1714
-1720,1736
+1700,1715
+171F,1736
 1740,1753
 1760,176C
 176E,1770
@@ -194,8 +192,7 @@ FCE,FDA
 1780,17DD
 17E0,17E9
 17F0,17F9
-1800,180E
-1810,1819
+1800,1819
 1820,1878
 1880,18AA
 18B0,18F5
@@ -214,9 +211,9 @@ FCE,FDA
 1A7F,1A89
 1A90,1A99
 1AA0,1AAD
-1AB0,1ABE
-1B00,1B4B
-1B50,1B7C
+1AB0,1ACE
+1B00,1B4C
+1B50,1B7E
 1B80,1BF3
 1BFC,1C37
 1C3B,1C49
@@ -224,8 +221,7 @@ FCE,FDA
 1C90,1CBA
 1CBD,1CC7
 1CD0,1CFA
-1D00,1DF9
-1DFB,1F15
+1D00,1F15
 1F18,1F1D
 1F20,1F45
 1F48,1F4D
@@ -245,16 +241,14 @@ FCE,FDA
 2066,2071
 2074,208E
 2090,209C
-20A0,20BF
+20A0,20C0
 20D0,20F0
 2100,218B
 2190,2426
 2440,244A
 2460,2B73
 2B76,2B95
-2B98,2C2E
-2C30,2C5E
-2C60,2CF3
+2B97,2CF3
 2CF9,2D25
 2D27,2D27
 2D2D,2D2D
@@ -269,7 +263,7 @@ FCE,FDA
 2DC8,2DCE
 2DD0,2DD6
 2DD8,2DDE
-2DE0,2E4F
+2DE0,2E5D
 2E80,2E99
 2E9B,2EF3
 2F00,2FD5
@@ -279,18 +273,17 @@ FCE,FDA
 3099,30FF
 3105,312F
 3131,318E
-3190,31BA
-31C0,31E3
+3190,31E3
 31F0,321E
-3220,4DB5
-4DC0,9FEF
-A000,A48C
+3220,A48C
 A490,A4C6
 A4D0,A62B
 A640,A6F7
-A700,A7BF
-A7C2,A7C6
-A7F7,A82B
+A700,A7CA
+A7D0,A7D1
+A7D3,A7D3
+A7D5,A7D9
+A7F2,A82C
 A830,A839
 A840,A877
 A880,A8C5
@@ -310,7 +303,7 @@ AB09,AB0E
 AB11,AB16
 AB20,AB26
 AB28,AB2E
-AB30,AB67
+AB30,AB6B
 AB70,ABED
 ABF0,ABF9
 AC00,D7A3
@@ -325,12 +318,11 @@ FB38,FB3C
 FB3E,FB3E
 FB40,FB41
 FB43,FB44
-FB46,FBC1
-FBD3,FD3F
-FD50,FD8F
+FB46,FBC2
+FBD3,FD8F
 FD92,FDC7
-FDF0,FDFD
-FE00,FE19
+FDCF,FDCF
+FDF0,FE19
 FE20,FE52
 FE54,FE66
 FE68,FE6B
@@ -355,7 +347,7 @@ FFF9,FFFD
 10100,10102
 10107,10133
 10137,1018E
-10190,1019B
+10190,1019C
 101A0,101A0
 101D0,101FD
 10280,1029C
@@ -373,10 +365,20 @@ FFF9,FFFD
 104D8,104FB
 10500,10527
 10530,10563
-1056F,1056F
+1056F,1057A
+1057C,1058A
+1058C,10592
+10594,10595
+10597,105A1
+105A3,105B1
+105B3,105B9
+105BB,105BC
 10600,10736
 10740,10755
 10760,10767
+10780,10785
+10787,107B0
+107B2,107BA
 10800,10805
 10808,10808
 1080A,10835
@@ -415,20 +417,24 @@ FFF9,FFFD
 10CFA,10D27
 10D30,10D39
 10E60,10E7E
+10E80,10EA9
+10EAB,10EAD
+10EB0,10EB1
 10F00,10F27
 10F30,10F59
+10F70,10F89
+10FB0,10FCB
 10FE0,10FF6
 11000,1104D
-11052,1106F
-1107F,110C1
+11052,11075
+1107F,110C2
 110CD,110CD
 110D0,110E8
 110F0,110F9
 11100,11134
-11136,11146
+11136,11147
 11150,11176
-11180,111CD
-111D0,111DF
+11180,111DF
 111E1,111F4
 11200,11211
 11213,1123E
@@ -454,9 +460,8 @@ FFF9,FFFD
 1135D,11363
 11366,1136C
 11370,11374
-11400,11459
-1145B,1145B
-1145D,1145F
+11400,1145B
+1145D,11461
 11480,114C7
 114D0,114D9
 11580,115B5
@@ -464,20 +469,27 @@ FFF9,FFFD
 11600,11644
 11650,11659
 11660,1166C
-11680,116B8
+11680,116B9
 116C0,116C9
 11700,1171A
 1171D,1172B
-11730,1173F
+11730,11746
 11800,1183B
 118A0,118F2
-118FF,118FF
+118FF,11906
+11909,11909
+1190C,11913
+11915,11916
+11918,11935
+11937,11938
+1193B,11946
+11950,11959
 119A0,119A7
 119AA,119D7
 119DA,119E4
 11A00,11A47
 11A50,11AA2
-11AC0,11AF8
+11AB0,11AF8
 11C00,11C08
 11C0A,11C36
 11C38,11C45
@@ -499,18 +511,21 @@ FFF9,FFFD
 11D93,11D98
 11DA0,11DA9
 11EE0,11EF8
+11FB0,11FB0
 11FC0,11FF1
 11FFF,12399
 12400,1246E
 12470,12474
 12480,12543
+12F90,12FF2
 13000,1342E
 13430,13438
 14400,14646
 16800,16A38
 16A40,16A5E
 16A60,16A69
-16A6E,16A6F
+16A6E,16ABE
+16AC0,16AC9
 16AD0,16AED
 16AF0,16AF5
 16B00,16B45
@@ -522,10 +537,15 @@ FFF9,FFFD
 16F00,16F4A
 16F4F,16F87
 16F8F,16F9F
-16FE0,16FE3
+16FE0,16FE4
+16FF0,16FF1
 17000,187F7
-18800,18AF2
-1B000,1B11E
+18800,18CD5
+18D00,18D08
+1AFF0,1AFF3
+1AFF5,1AFFB
+1AFFD,1AFFE
+1B000,1B122
 1B150,1B152
 1B164,1B167
 1B170,1B2FB
@@ -534,9 +554,12 @@ FFF9,FFFD
 1BC80,1BC88
 1BC90,1BC99
 1BC9C,1BCA3
+1CF00,1CF2D
+1CF30,1CF46
+1CF50,1CFC3
 1D000,1D0F5
 1D100,1D126
-1D129,1D1E8
+1D129,1D1EA
 1D200,1D245
 1D2E0,1D2F3
 1D300,1D356
@@ -564,6 +587,7 @@ FFF9,FFFD
 1D7CE,1DA8B
 1DA9B,1DA9F
 1DAA1,1DAAF
+1DF00,1DF1E
 1E000,1E006
 1E008,1E018
 1E01B,1E021
@@ -573,8 +597,13 @@ FFF9,FFFD
 1E130,1E13D
 1E140,1E149
 1E14E,1E14F
+1E290,1E2AE
 1E2C0,1E2F9
 1E2FF,1E2FF
+1E7E0,1E7E6
+1E7E8,1E7EB
+1E7ED,1E7EE
+1E7F0,1E7FE
 1E800,1E8C4
 1E8C7,1E8D6
 1E900,1E94B
@@ -622,43 +651,46 @@ FFF9,FFFD
 1F0B1,1F0BF
 1F0C1,1F0CF
 1F0D1,1F0F5
-1F100,1F10C
-1F110,1F16C
-1F170,1F1AC
+1F100,1F1AD
 1F1E6,1F202
 1F210,1F23B
 1F240,1F248
 1F250,1F251
 1F260,1F265
-1F300,1F6D5
-1F6E0,1F6EC
-1F6F0,1F6FA
+1F300,1F6D7
+1F6DD,1F6EC
+1F6F0,1F6FC
 1F700,1F773
 1F780,1F7D8
 1F7E0,1F7EB
+1F7F0,1F7F0
 1F800,1F80B
 1F810,1F847
 1F850,1F859
 1F860,1F887
 1F890,1F8AD
-1F900,1F90B
-1F90D,1F971
-1F973,1F976
-1F97A,1F9A2
-1F9A5,1F9AA
-1F9AE,1F9CA
-1F9CD,1FA53
+1F8B0,1F8B1
+1F900,1FA53
 1FA60,1FA6D
-1FA70,1FA73
-1FA78,1FA7A
-1FA80,1FA82
-1FA90,1FA95
-20000,2A6D6
-2A700,2B734
+1FA70,1FA74
+1FA78,1FA7C
+1FA80,1FA86
+1FA90,1FAAC
+1FAB0,1FABA
+1FAC0,1FAC5
+1FAD0,1FAD9
+1FAE0,1FAE7
+1FAF0,1FAF6
+1FB00,1FB92
+1FB94,1FBCA
+1FBF0,1FBF9
+20000,2A6DF
+2A700,2B738
 2B740,2B81D
 2B820,2CEA1
 2CEB0,2EBE0
 2F800,2FA1D
+30000,3134A
 E0001,E0001
 E0020,E007F
 E0100,E01EF

data/lib/character_set/predefined_sets/emoji.cps

@@ -44,6 +44,7 @@ AE,AE
 2699,2699
 269B,269C
 26A0,26A1
+26A7,26A7
 26AA,26AB
 26B0,26B1
 26BD,26BE
@@ -130,22 +131,23 @@ AE,AE
 1F5FA,1F64F
 1F680,1F6C5
 1F6CB,1F6D2
-1F6D5,1F6D5
-1F6E0,1F6E5
+1F6D5,1F6D7
+1F6DD,1F6E5
 1F6E9,1F6E9
 1F6EB,1F6EC
 1F6F0,1F6F0
-1F6F3,1F6FA
+1F6F3,1F6FC
 1F7E0,1F7EB
-1F90D,1F93A
+1F7F0,1F7F0
+1F90C,1F93A
 1F93C,1F945
-1F947,1F971
-1F973,1F976
-1F97A,1F9A2
-1F9A5,1F9AA
-1F9AE,1F9CA
-1F9CD,1F9FF
-1FA70,1FA73
-1FA78,1FA7A
-1FA80,1FA82
-1FA90,1FA95
+1F947,1F9FF
+1FA70,1FA74
+1FA78,1FA7C
+1FA80,1FA86
+1FA90,1FAAC
+1FAB0,1FABA
+1FAC0,1FAC5
+1FAD0,1FAD9
+1FAE0,1FAE7
+1FAF0,1FAF6

data/lib/character_set/predefined_sets.rb

@@ -22,6 +22,17 @@ class CharacterSet
     alias valid                    unicode
 
     def build_from_cps_file(path)
+      if defined?(Ractor) && Ractor.current != Ractor.main
+        raise <<-EOS.gsub(/^ */, '')
+          CharacterSet's predefined sets are lazy-loaded.
+          Pre-load them to use them in Ractors. E.g.:
+
+          CharacterSet.ascii # pre-load
+          Ractor.new { CharacterSet.ascii.size }.take # => 128
+          Ractor.new { 'abc'.keep_character_set(:ascii) }.take # => 'abc'
+        EOS
+      end
+
       File.readlines(path).inject(new) do |set, line|
         range_start, range_end = line.split(',')
         set.merge((range_start.to_i(16))..(range_end.to_i(16)))

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -6,9 +6,9 @@ class CharacterSet
           new(Array(ranges).flat_map(&:to_a))
         end
 
-        def of(string)
-          raise ArgumentError, 'pass a String' unless string.is_a?(String)
-          new(string.codepoints)
+        def of_string(str)
+          raise ArgumentError, 'pass a String' unless str.respond_to?(:codepoints)
+          str.encode('utf-8').each_codepoint.with_object(new) { |cp, set| set << cp }
         end
       end
 
@@ -31,7 +31,7 @@ class CharacterSet
       end
 
       def ranges
-        CharacterSet.require_optional_dependency('range_compressor')
+        CharacterSet.require_optional_dependency('range_compressor', __method__)
         RangeCompressor.compress(self)
       end
 
@@ -40,16 +40,18 @@ class CharacterSet
       end
 
       def count_in(string)
-        str!(string).each_codepoint.count { |cp| include?(cp) }
+        utf8_str!(string).each_codepoint.count { |cp| include?(cp) }
       end
 
       def cover?(string)
-        str!(string).each_codepoint { |cp| return false unless include?(cp) }
+        utf8_str!(string).each_codepoint { |cp| return false unless include?(cp) }
         true
       end
 
       def delete_in(string)
-        make_new_str(string) { |cp, new_str| include?(cp) || (new_str << cp) }
+        utf8_str!(string).each_codepoint.with_object('') do |cp, new_str|
+          include?(cp) || (new_str << cp)
+        end.encode(string.encoding)
       end
 
       def delete_in!(string)
@@ -58,7 +60,9 @@ class CharacterSet
       end
 
       def keep_in(string)
-        make_new_str(string) { |cp, new_str| include?(cp) && (new_str << cp) }
+        utf8_str!(string).each_codepoint.with_object('') do |cp, new_str|
+          include?(cp) && (new_str << cp)
+        end.encode(string.encoding)
       end
 
       def keep_in!(string)
@@ -67,14 +71,13 @@ class CharacterSet
       end
 
       def scan(string)
-        encoding = str!(string).encoding
-        string.each_codepoint.inject([]) do |arr, cp|
-          include?(cp) ? arr.push(cp.chr(encoding)) : arr
+        utf8_str!(string).each_codepoint.with_object([]) do |cp, arr|
+          arr.push(cp.chr('utf-8')) if include?(cp)
         end
       end
 
       def used_by?(string)
-        str!(string).each_codepoint { |cp| return true if include?(cp) }
+        utf8_str!(string).each_codepoint { |cp| return true if include?(cp) }
         false
       end
 
@@ -115,16 +118,9 @@ class CharacterSet
         num >= 0 && num <= 16 or raise ArgumentError, 'plane must be between 0 and 16'
       end
 
-      def str!(obj)
+      def utf8_str!(obj)
         raise ArgumentError, 'pass a String' unless obj.respond_to?(:codepoints)
-        obj
-      end
-
-      def make_new_str(original, &block)
-        new_string = str!(original)
-          .each_codepoint
-          .each_with_object(''.encode(original.encoding), &block)
-        original.tainted? ? new_string.taint : new_string
+        obj.encode('utf-8')
       end
     end
   end

data/lib/character_set/ruby_fallback/set_methods.rb

@@ -1,7 +1,9 @@
 class CharacterSet
   module RubyFallback
     module SetMethods
-      Enumerable.instance_methods.concat(%w[empty? length size]).each do |mthd|
+      (Enumerable.instance_methods -
+        %i[include? member? to_a] +
+        %i[empty? hash length size]).each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(*args, &block)
             @__set.#{mthd}(*args, &block)
@@ -9,8 +11,8 @@ class CharacterSet
         RUBY
       end
 
-      %w[< <= > >= disjoint? intersect? proper_subset? proper_superset?
-         subset? superset?].each do |mthd|
+      %i[< <= <=> > >= === disjoint? include? intersect? member?
+         proper_subset? proper_superset? subset? superset?].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(enum, &block)
             if enum.is_a?(CharacterSet) || enum.is_a?(CharacterSet::Pure)
@@ -21,9 +23,8 @@ class CharacterSet
         RUBY
       end
 
-      %w[<< === add add? clear collect! delete delete? delete_if
-         each filter! hash include? map! member? keep_if reject!
-         select! subtract].each do |mthd|
+      %i[<< add add? clear delete delete? delete_if each filter! keep_if
+         reject! select! subtract].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(*args, &block)
             result = @__set.#{mthd}(*args, &block)
@@ -32,7 +33,7 @@ class CharacterSet
         RUBY
       end
 
-      %w[& + - ^ | difference intersection union].each do |mthd|
+      %i[& + - ^ | difference intersection union].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(enum, &block)
             if enum.respond_to?(:map)
@@ -43,15 +44,6 @@ class CharacterSet
         RUBY
       end
 
-      %w[taint untaint].each do |mthd|
-        class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{mthd}
-            @__set.#{mthd}
-            super
-          end
-        RUBY
-      end
-
       unless RUBY_PLATFORM[/java/i]
         def freeze
           @__set.to_a
@@ -81,6 +73,7 @@ class CharacterSet
 
       def eql?(other)
         return false unless other.is_a?(self.class)
+
         @__set.eql?(other.instance_variable_get(:@__set))
       end