character_set 1.4.1 → 1.6.0

data/ext/character_set/unicode_casefold_table.h

@@ -6,7 +6,7 @@ typedef struct casefold_mapping {
   unsigned long to;
 } casefold_mapping;
 
-#define CASEFOLD_COUNT 1383
+#define CASEFOLD_COUNT 1426
 
 static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x0041,0x0061},
@@ -564,6 +564,41 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x104D1,0x104F9},
 {0x104D2,0x104FA},
 {0x104D3,0x104FB},
+{0x10570,0x10597},
+{0x10571,0x10598},
+{0x10572,0x10599},
+{0x10573,0x1059A},
+{0x10574,0x1059B},
+{0x10575,0x1059C},
+{0x10576,0x1059D},
+{0x10577,0x1059E},
+{0x10578,0x1059F},
+{0x10579,0x105A0},
+{0x1057A,0x105A1},
+{0x1057C,0x105A3},
+{0x1057D,0x105A4},
+{0x1057E,0x105A5},
+{0x1057F,0x105A6},
+{0x10580,0x105A7},
+{0x10581,0x105A8},
+{0x10582,0x105A9},
+{0x10583,0x105AA},
+{0x10584,0x105AB},
+{0x10585,0x105AC},
+{0x10586,0x105AD},
+{0x10587,0x105AE},
+{0x10588,0x105AF},
+{0x10589,0x105B0},
+{0x1058A,0x105B1},
+{0x1058C,0x105B3},
+{0x1058D,0x105B4},
+{0x1058E,0x105B5},
+{0x1058F,0x105B6},
+{0x10590,0x105B7},
+{0x10591,0x105B8},
+{0x10592,0x105B9},
+{0x10594,0x105BB},
+{0x10595,0x105BC},
 {0x10A0,0x2D00},
 {0x10A1,0x2D01},
 {0x10A2,0x2D02},
@@ -1102,6 +1137,7 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x2C2C,0x2C5C},
 {0x2C2D,0x2C5D},
 {0x2C2E,0x2C5E},
+{0x2C2F,0x2C5F},
 {0x2C60,0x2C61},
 {0x2C62,0x026B},
 {0x2C63,0x1D7D},
@@ -1282,10 +1318,17 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0xA7BA,0xA7BB},
 {0xA7BC,0xA7BD},
 {0xA7BE,0xA7BF},
+{0xA7C0,0xA7C1},
 {0xA7C2,0xA7C3},
 {0xA7C4,0xA794},
 {0xA7C5,0x0282},
 {0xA7C6,0x1D8E},
+{0xA7C7,0xA7C8},
+{0xA7C9,0xA7CA},
+{0xA7D0,0xA7D1},
+{0xA7D6,0xA7D7},
+{0xA7D8,0xA7D9},
+{0xA7F5,0xA7F6},
 {0xAB70,0x13A0},
 {0xAB71,0x13A1},
 {0xAB72,0x13A2},

data/lib/character_set/core_ext/string_ext.rb

@@ -2,7 +2,7 @@ class CharacterSet
   module CoreExt
     module StringExt
       def character_set
-        CharacterSet.of(self)
+        CharacterSet.of_string(self)
       end
 
       {

data/lib/character_set/expression_converter.rb

@@ -4,7 +4,7 @@ class CharacterSet
 
     Error = Class.new(ArgumentError)
 
-    def convert(expression)
+    def convert(expression, to = CharacterSet)
       CharacterSet.require_optional_dependency('regexp_parser', __method__)
 
       case expression
@@ -12,49 +12,49 @@ class CharacterSet
         if expression.count != 1
           raise Error, 'Pass a Regexp with exactly one expression, e.g. /[a-z]/'
         end
-        convert(expression[0])
+        convert(expression[0], to)
 
       when Regexp::Expression::CharacterSet
-        content = expression.map { |subexp| convert(subexp) }.reduce(:+)
-        content ||= CharacterSet[]
+        content = expression.map { |subexp| convert(subexp, to) }.reduce(:+)
+        content ||= to[]
         expression.negative? ? content.inversion : content
 
       when Regexp::Expression::CharacterSet::Intersection
-        expression.map { |subexp| convert(subexp) }.reduce(:&)
+        expression.map { |subexp| convert(subexp, to) }.reduce(:&)
 
       when Regexp::Expression::CharacterSet::IntersectedSequence
-        expression.map { |subexp| convert(subexp) }.reduce(:+) || CharacterSet[]
+        expression.map { |subexp| convert(subexp, to) }.reduce(:+) || to[]
 
       when Regexp::Expression::CharacterSet::Range
-        start, finish = expression.map { |subexp| convert(subexp) }
-        CharacterSet.new((start.min)..(finish.max))
+        start, finish = expression.map { |subexp| convert(subexp, to) }
+        to.new((start.min)..(finish.max))
 
       when Regexp::Expression::CharacterType::Any
-        CharacterSet.unicode
+        to.unicode
 
       when Regexp::Expression::CharacterType::Base
         /(?<negative>non)?(?<base_name>.+)/ =~ expression.token
         content =
           if expression.unicode_classes?
             # in u-mode, type shortcuts match the same as \p{<long type name>}
-            CharacterSet.of_property(base_name)
+            to.of_property(base_name)
           else
             # in normal mode, types match only ascii chars
             case base_name.to_sym
-            when :digit then CharacterSet.from_ranges(48..57)
-            when :hex   then CharacterSet.from_ranges(48..57, 65..70, 97..102)
-            when :space then CharacterSet.from_ranges(9..13, 32..32)
-            when :word  then CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122)
+            when :digit then to.from_ranges(48..57)
+            when :hex   then to.from_ranges(48..57, 65..70, 97..102)
+            when :space then to.from_ranges(9..13, 32..32)
+            when :word  then to.from_ranges(48..57, 65..90, 95..95, 97..122)
             else raise Error, "Unsupported CharacterType #{base_name}"
             end
           end
         negative ? content.inversion : content
 
       when Regexp::Expression::EscapeSequence::CodepointList
-        CharacterSet.new(expression.codepoints)
+        to.new(expression.codepoints)
 
       when Regexp::Expression::EscapeSequence::Base
-        CharacterSet[expression.codepoint]
+        to[expression.codepoint]
 
       when Regexp::Expression::Group::Capture,
            Regexp::Expression::Group::Passive,
@@ -62,19 +62,19 @@ class CharacterSet
            Regexp::Expression::Group::Atomic,
            Regexp::Expression::Group::Options
         case expression.count
-        when 0 then CharacterSet[]
-        when 1 then convert(expression.first)
+        when 0 then to[]
+        when 1 then convert(expression.first, to)
         else
           raise Error, 'Groups must contain exactly one expression, e.g. ([a-z])'
         end
 
       when Regexp::Expression::Alternation # rubocop:disable Lint/DuplicateBranch
-        expression.map { |subexp| convert(subexp) }.reduce(:+)
+        expression.map { |subexp| convert(subexp, to) }.reduce(:+)
 
       when Regexp::Expression::Alternative
         case expression.count
-        when 0 then CharacterSet[]
-        when 1 then convert(expression.first)
+        when 0 then to[]
+        when 1 then convert(expression.first, to)
         else
           raise Error, 'Alternatives must contain exactly one expression'
         end
@@ -83,11 +83,11 @@ class CharacterSet
         if expression.set_level == 0 && expression.text.size != 1
           raise Error, 'Literal runs outside of sets are codepoint *sequences*'
         end
-        CharacterSet[expression.text.ord]
+        to[expression.text.ord]
 
       when Regexp::Expression::UnicodeProperty::Base,
            Regexp::Expression::PosixClass
-        content = CharacterSet.of_property(expression.token)
+        content = to.of_property(expression.token)
         if expression.type == :posixclass && expression.ascii_classes?
           content = content.ascii_part
         end

data/lib/character_set/predefined_sets/assigned.cps

@@ -21,7 +21,7 @@
 85E,85E
 860,86A
 8A0,8B4
-8B6,8BD
+8B6,8C7
 8D3,983
 985,98C
 98F,990
@@ -76,7 +76,7 @@ B35,B39
 B3C,B44
 B47,B48
 B4B,B4D
-B56,B57
+B55,B57
 B5C,B5D
 B5F,B63
 B66,B77
@@ -120,15 +120,14 @@ CDE,CDE
 CE0,CE3
 CE6,CEF
 CF1,CF2
-D00,D03
-D05,D0C
+D00,D0C
 D0E,D10
 D12,D44
 D46,D48
 D4A,D4F
 D54,D63
 D66,D7F
-D82,D83
+D81,D83
 D85,D96
 D9A,DB1
 DB3,DBB
@@ -214,7 +213,7 @@ FCE,FDA
 1A7F,1A89
 1A90,1A99
 1AA0,1AAD
-1AB0,1ABE
+1AB0,1AC0
 1B00,1B4B
 1B50,1B7C
 1B80,1BF3
@@ -252,7 +251,7 @@ FCE,FDA
 2440,244A
 2460,2B73
 2B76,2B95
-2B98,2C2E
+2B97,2C2E
 2C30,2C5E
 2C60,2CF3
 2CF9,2D25
@@ -269,7 +268,7 @@ FCE,FDA
 2DC8,2DCE
 2DD0,2DD6
 2DD8,2DDE
-2DE0,2E4F
+2DE0,2E52
 2E80,2E99
 2E9B,2EF3
 2F00,2FD5
@@ -279,18 +278,16 @@ FCE,FDA
 3099,30FF
 3105,312F
 3131,318E
-3190,31BA
-31C0,31E3
+3190,31E3
 31F0,321E
-3220,4DB5
-4DC0,9FEF
+3220,9FFC
 A000,A48C
 A490,A4C6
 A4D0,A62B
 A640,A6F7
 A700,A7BF
-A7C2,A7C6
-A7F7,A82B
+A7C2,A7CA
+A7F5,A82C
 A830,A839
 A840,A877
 A880,A8C5
@@ -310,7 +307,7 @@ AB09,AB0E
 AB11,AB16
 AB20,AB26
 AB28,AB2E
-AB30,AB67
+AB30,AB6B
 AB70,ABED
 ABF0,ABF9
 AC00,D7A3
@@ -355,7 +352,7 @@ FFF9,FFFD
 10100,10102
 10107,10133
 10137,1018E
-10190,1019B
+10190,1019C
 101A0,101A0
 101D0,101FD
 10280,1029C
@@ -415,8 +412,12 @@ FFF9,FFFD
 10CFA,10D27
 10D30,10D39
 10E60,10E7E
+10E80,10EA9
+10EAB,10EAD
+10EB0,10EB1
 10F00,10F27
 10F30,10F59
+10FB0,10FCB
 10FE0,10FF6
 11000,1104D
 11052,1106F
@@ -425,10 +426,9 @@ FFF9,FFFD
 110D0,110E8
 110F0,110F9
 11100,11134
-11136,11146
+11136,11147
 11150,11176
-11180,111CD
-111D0,111DF
+11180,111DF
 111E1,111F4
 11200,11211
 11213,1123E
@@ -454,9 +454,8 @@ FFF9,FFFD
 1135D,11363
 11366,1136C
 11370,11374
-11400,11459
-1145B,1145B
-1145D,1145F
+11400,1145B
+1145D,11461
 11480,114C7
 114D0,114D9
 11580,115B5
@@ -471,7 +470,14 @@ FFF9,FFFD
 11730,1173F
 11800,1183B
 118A0,118F2
-118FF,118FF
+118FF,11906
+11909,11909
+1190C,11913
+11915,11916
+11918,11935
+11937,11938
+1193B,11946
+11950,11959
 119A0,119A7
 119AA,119D7
 119DA,119E4
@@ -499,6 +505,7 @@ FFF9,FFFD
 11D93,11D98
 11DA0,11DA9
 11EE0,11EF8
+11FB0,11FB0
 11FC0,11FF1
 11FFF,12399
 12400,1246E
@@ -522,9 +529,11 @@ FFF9,FFFD
 16F00,16F4A
 16F4F,16F87
 16F8F,16F9F
-16FE0,16FE3
+16FE0,16FE4
+16FF0,16FF1
 17000,187F7
-18800,18AF2
+18800,18CD5
+18D00,18D08
 1B000,1B11E
 1B150,1B152
 1B164,1B167
@@ -622,17 +631,15 @@ FFF9,FFFD
 1F0B1,1F0BF
 1F0C1,1F0CF
 1F0D1,1F0F5
-1F100,1F10C
-1F110,1F16C
-1F170,1F1AC
+1F100,1F1AD
 1F1E6,1F202
 1F210,1F23B
 1F240,1F248
 1F250,1F251
 1F260,1F265
-1F300,1F6D5
+1F300,1F6D7
 1F6E0,1F6EC
-1F6F0,1F6FA
+1F6F0,1F6FC
 1F700,1F773
 1F780,1F7D8
 1F7E0,1F7EB
@@ -641,24 +648,28 @@ FFF9,FFFD
 1F850,1F859
 1F860,1F887
 1F890,1F8AD
-1F900,1F90B
-1F90D,1F971
-1F973,1F976
-1F97A,1F9A2
-1F9A5,1F9AA
-1F9AE,1F9CA
+1F8B0,1F8B1
+1F900,1F978
+1F97A,1F9CB
 1F9CD,1FA53
 1FA60,1FA6D
-1FA70,1FA73
+1FA70,1FA74
 1FA78,1FA7A
-1FA80,1FA82
-1FA90,1FA95
-20000,2A6D6
+1FA80,1FA86
+1FA90,1FAA8
+1FAB0,1FAB6
+1FAC0,1FAC2
+1FAD0,1FAD6
+1FB00,1FB92
+1FB94,1FBCA
+1FBF0,1FBF9
+20000,2A6DD
 2A700,2B734
 2B740,2B81D
 2B820,2CEA1
 2CEB0,2EBE0
 2F800,2FA1D
+30000,3134A
 E0001,E0001
 E0020,E007F
 E0100,E01EF

data/lib/character_set/predefined_sets/emoji.cps

@@ -44,6 +44,7 @@ AE,AE
 2699,2699
 269B,269C
 26A0,26A1
+26A7,26A7
 26AA,26AB
 26B0,26B1
 26BD,26BE
@@ -130,22 +131,22 @@ AE,AE
 1F5FA,1F64F
 1F680,1F6C5
 1F6CB,1F6D2
-1F6D5,1F6D5
+1F6D5,1F6D7
 1F6E0,1F6E5
 1F6E9,1F6E9
 1F6EB,1F6EC
 1F6F0,1F6F0
-1F6F3,1F6FA
+1F6F3,1F6FC
 1F7E0,1F7EB
-1F90D,1F93A
+1F90C,1F93A
 1F93C,1F945
-1F947,1F971
-1F973,1F976
-1F97A,1F9A2
-1F9A5,1F9AA
-1F9AE,1F9CA
+1F947,1F978
+1F97A,1F9CB
 1F9CD,1F9FF
-1FA70,1FA73
+1FA70,1FA74
 1FA78,1FA7A
-1FA80,1FA82
-1FA90,1FA95
+1FA80,1FA86
+1FA90,1FAA8
+1FAB0,1FAB6
+1FAC0,1FAC2
+1FAD0,1FAD6

data/lib/character_set/predefined_sets.rb

@@ -22,6 +22,17 @@ class CharacterSet
     alias valid                    unicode
 
     def build_from_cps_file(path)
+      if defined?(Ractor) && Ractor.current != Ractor.main
+        raise <<-EOS.gsub(/^ */, '')
+          CharacterSet's predefined sets are lazy-loaded.
+          Pre-load them to use them in Ractors. E.g.:
+
+          CharacterSet.ascii # pre-load
+          Ractor.new { CharacterSet.ascii.size }.take # => 128
+          Ractor.new { 'abc'.keep_character_set(:ascii) }.take # => 'abc'
+        EOS
+      end
+
       File.readlines(path).inject(new) do |set, line|
         range_start, range_end = line.split(',')
         set.merge((range_start.to_i(16))..(range_end.to_i(16)))

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -6,9 +6,9 @@ class CharacterSet
           new(Array(ranges).flat_map(&:to_a))
         end
 
-        def of(string)
-          raise ArgumentError, 'pass a String' unless string.is_a?(String)
-          new(string.codepoints)
+        def of_string(str)
+          raise ArgumentError, 'pass a String' unless str.respond_to?(:codepoints)
+          str.codepoints.each_with_object(new) { |cp, set| set << cp }
         end
       end
 

data/lib/character_set/set_method_adapters.rb

@@ -22,13 +22,14 @@ class CharacterSet
 
     # Allow some methods to take an Enum just as well as another CharacterSet.
     # Tested by ruby-spec.
-    %w[& + - ^ | difference intersection subtract union].each do |method|
+    %w[& + - ^ | difference disjoint? intersect? intersection
+       subtract union].each do |method|
       class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{method}(arg)
           if arg.is_a?(CharacterSet)
-            super
+            super(arg)
           elsif arg.respond_to?(:each)
-            super(CharacterSet.new(arg.to_a))
+            super(self.class.new(arg.to_a))
           else
             raise ArgumentError, 'pass an enumerable'
           end

data/lib/character_set/shared_methods.rb

@@ -15,6 +15,12 @@ class CharacterSet
             new(Array(args))
           end
 
+          def of(*args)
+            args.map do |arg|
+              arg.is_a?(Regexp) ? of_regexp(arg) : of_string(arg)
+            end.reduce(:merge) || new
+          end
+
           def parse(string)
             codepoints = Parser.codepoints_from_bracket_expression(string)
             result = new(codepoints)
@@ -36,7 +42,7 @@ class CharacterSet
           end
 
           def of_expression(expression)
-            ExpressionConverter.convert(expression)
+            ExpressionConverter.convert(expression, self)
           end
 
           def require_optional_dependency(name, method)
@@ -90,6 +96,14 @@ class CharacterSet
           Writer.write_surrogate_alternation(bmp_part.ranges, astral_part.ranges)
         end
 
+        def secure_token(length = 32)
+          CharacterSet.require_optional_dependency('securerandom', __method__)
+          cps = to_a
+          len = cps.count
+          1.upto(length).map { cps[SecureRandom.random_number(len)] }.pack('U*')
+        end
+        alias random_token secure_token
+
         def inspect
           len = length
           "#<#{klass.name}: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"

data/lib/character_set/version.rb

@@ -1,3 +1,3 @@
 class CharacterSet
-  VERSION = '1.4.1'
+  VERSION = '1.6.0'
 end