character_set 1.7.0 → 1.9.0

data/lib/character_set/expression_converter.rb

@@ -4,86 +4,61 @@ class CharacterSet
 
     Error = Class.new(ArgumentError)
 
-    def convert(expression, to = CharacterSet)
+    def convert(expression, to = CharacterSet, acc = [])
       CharacterSet.require_optional_dependency('regexp_parser', __method__)
 
       case expression
-      when Regexp::Expression::Root
-        if expression.count != 1
-          raise Error, 'Pass a Regexp with exactly one expression, e.g. /[a-z]/'
-        end
-        convert(expression[0], to)
-
       when Regexp::Expression::CharacterSet
-        content = expression.map { |subexp| convert(subexp, to) }.reduce(:+)
-        content ||= to[]
-        expression.negative? ? content.inversion : content
+        content = expression.map { |subexp| convert(subexp, to) }.reduce(:+) || to[]
+        acc << (expression.negative? ? content.inversion : content)
 
       when Regexp::Expression::CharacterSet::Intersection
-        expression.map { |subexp| convert(subexp, to) }.reduce(:&)
-
-      when Regexp::Expression::CharacterSet::IntersectedSequence
-        expression.map { |subexp| convert(subexp, to) }.reduce(:+) || to[]
+        acc << expression.map { |subexp| convert(subexp, to) }.reduce(:&)
 
       when Regexp::Expression::CharacterSet::Range
         start, finish = expression.map { |subexp| convert(subexp, to) }
-        to.new((start.min)..(finish.max))
+        acc << to.new((start.min)..(finish.max))
+
+      when Regexp::Expression::Subexpression # root, group, alternation, etc.
+        expression.each { |subexp| convert(subexp, to, acc) }
 
       when Regexp::Expression::CharacterType::Any
-        to.unicode
+        acc << to.unicode
 
       when Regexp::Expression::CharacterType::Base
         /(?<negative>non)?(?<base_name>.+)/ =~ expression.token
         content =
           if expression.unicode_classes?
-            # in u-mode, type shortcuts match the same as \p{<long type name>}
-            to.of_property(base_name)
+            # in u-mode, most type shortcuts match the same as \p{<long type name>}
+            if base_name == 'linebreak'
+              to.from_ranges(10..13, 133..133, 8232..8233)
+            else
+              to.of_property(base_name)
+            end
           else
             # in normal mode, types match only ascii chars
             case base_name.to_sym
-            when :digit then to.from_ranges(48..57)
-            when :hex   then to.from_ranges(48..57, 65..70, 97..102)
-            when :space then to.from_ranges(9..13, 32..32)
-            when :word  then to.from_ranges(48..57, 65..90, 95..95, 97..122)
+            when :digit     then to.from_ranges(48..57)
+            when :hex       then to.from_ranges(48..57, 65..70, 97..102)
+            when :linebreak then to.from_ranges(10..13)
+            when :space     then to.from_ranges(9..13, 32..32)
+            when :word      then to.from_ranges(48..57, 65..90, 95..95, 97..122)
             else raise Error, "Unsupported CharacterType #{base_name}"
             end
           end
-        negative ? content.inversion : content
+        acc << (negative ? content.inversion : content)
 
       when Regexp::Expression::EscapeSequence::CodepointList
-        to.new(expression.codepoints)
+        content = to.new(expression.codepoints)
+        acc << (expression.i? ? content.case_insensitive : content)
 
       when Regexp::Expression::EscapeSequence::Base
-        to[expression.codepoint]
-
-      when Regexp::Expression::Group::Capture,
-           Regexp::Expression::Group::Passive,
-           Regexp::Expression::Group::Named,
-           Regexp::Expression::Group::Atomic,
-           Regexp::Expression::Group::Options
-        case expression.count
-        when 0 then to[]
-        when 1 then convert(expression.first, to)
-        else
-          raise Error, 'Groups must contain exactly one expression, e.g. ([a-z])'
-        end
-
-      when Regexp::Expression::Alternation # rubocop:disable Lint/DuplicateBranch
-        expression.map { |subexp| convert(subexp, to) }.reduce(:+)
-
-      when Regexp::Expression::Alternative
-        case expression.count
-        when 0 then to[]
-        when 1 then convert(expression.first, to)
-        else
-          raise Error, 'Alternatives must contain exactly one expression'
-        end
+        content = to[expression.codepoint]
+        acc << (expression.i? ? content.case_insensitive : content)
 
       when Regexp::Expression::Literal
-        if expression.set_level == 0 && expression.text.size != 1
-          raise Error, 'Literal runs outside of sets are codepoint *sequences*'
-        end
-        to[expression.text.ord]
+        content = to[*expression.text.chars]
+        acc << (expression.i? ? content.case_insensitive : content)
 
       when Regexp::Expression::UnicodeProperty::Base,
            Regexp::Expression::PosixClass
@@ -91,14 +66,22 @@ class CharacterSet
         if expression.type == :posixclass && expression.ascii_classes?
           content = content.ascii_part
         end
-        expression.negative? ? content.inversion : content
+        acc << (expression.negative? ? content.inversion : content)
+
+      when Regexp::Expression::Anchor::Base,
+           Regexp::Expression::Backreference::Base,
+           Regexp::Expression::Keep::Mark,
+           Regexp::Expression::Quantifier
+        # ignore zero-length and repeat expressions
 
       when Regexp::Expression::Base
         raise Error, "Unsupported expression class `#{expression.class}`"
 
       else
-        raise Error, "Pass an expression (result of Regexp::Parser.parse)"
+        raise Error, 'Pass an expression (result of Regexp::Parser.parse)'
       end
+
+      acc.reduce(:+) || to[]
     end
   end
 end

data/lib/character_set/java.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'java'
+require_relative 'character_set.jar'
+
+JRuby::Util.load_ext('org.jaynetics.characterset.CharacterSetService')

data/lib/character_set/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CharacterSet
   module Parser
     module_function
@@ -18,7 +20,7 @@ class CharacterSet
 
     def codepoints_from_bracket_expression(string)
       raise ArgumentError, 'pass a String'   unless string.is_a?(String)
-      raise ArgumentError, 'advanced syntax' if string =~ /\\[^uUx]|[^\\]\[|&&/
+      raise ArgumentError, 'advanced syntax' if string.match?(/\\[^uUx]|[^\\]\[|&&/)
 
       content = strip_brackets(string)
       literal_content = eval_escapes(content)
@@ -26,7 +28,7 @@ class CharacterSet
       prev_chr = nil
       in_range = false
 
-      literal_content.each_char.map do |chr|
+      literal_content.each_char.filter_map do |chr|
         if chr == '-' && prev_chr && prev_chr != '\\' && prev_chr != '-'
           in_range = true
           nil
@@ -36,7 +38,7 @@ class CharacterSet
           prev_chr = chr
           result
         end
-      end.compact.flatten
+      end.flatten
     end
 
     def strip_brackets(string)

data/lib/character_set/predefined_sets/assigned.cps

@@ -19,9 +19,8 @@
 840,85B
 85E,85E
 860,86A
-870,88E
-890,891
-898,983
+870,891
+897,983
 985,98C
 98F,990
 993,9A8
@@ -104,7 +103,7 @@ C46,C48
 C4A,C4D
 C55,C56
 C58,C5A
-C5D,C5D
+C5C,C5D
 C60,C63
 C66,C6F
 C77,C8C
@@ -116,10 +115,10 @@ CBC,CC4
 CC6,CC8
 CCA,CCD
 CD5,CD6
-CDD,CDE
+CDC,CDE
 CE0,CE3
 CE6,CEF
-CF1,CF2
+CF1,CF3
 D00,D0C
 D0E,D10
 D12,D44
@@ -149,7 +148,7 @@ EA5,EA5
 EA7,EBD
 EC0,EC4
 EC6,EC6
-EC8,ECD
+EC8,ECE
 ED0,ED9
 EDC,EDF
 F00,F47
@@ -211,13 +210,13 @@ FCE,FDA
 1A7F,1A89
 1A90,1A99
 1AA0,1AAD
-1AB0,1ACE
+1AB0,1ADD
+1AE0,1AEB
 1B00,1B4C
-1B50,1B7E
-1B80,1BF3
+1B4E,1BF3
 1BFC,1C37
 1C3B,1C49
-1C4D,1C88
+1C4D,1C8A
 1C90,1CBA
 1CBD,1CC7
 1CD0,1CFA
@@ -241,14 +240,13 @@ FCE,FDA
 2066,2071
 2074,208E
 2090,209C
-20A0,20C0
+20A0,20C1
 20D0,20F0
 2100,218B
-2190,2426
+2190,2429
 2440,244A
 2460,2B73
-2B76,2B95
-2B97,2CF3
+2B76,2CF3
 2CF9,2D25
 2D27,2D27
 2D2D,2D2D
@@ -267,23 +265,19 @@ FCE,FDA
 2E80,2E99
 2E9B,2EF3
 2F00,2FD5
-2FF0,2FFB
-3000,303F
+2FF0,303F
 3041,3096
 3099,30FF
 3105,312F
 3131,318E
-3190,31E3
-31F0,321E
+3190,31E5
+31EF,321E
 3220,A48C
 A490,A4C6
 A4D0,A62B
 A640,A6F7
-A700,A7CA
-A7D0,A7D1
-A7D3,A7D3
-A7D5,A7D9
-A7F2,A82C
+A700,A7DC
+A7F1,A82C
 A830,A839
 A840,A877
 A880,A8C5
@@ -309,7 +303,7 @@ ABF0,ABF9
 AC00,D7A3
 D7B0,D7C6
 D7CB,D7FB
-D800,FA6D
+E000,FA6D
 FA70,FAD9
 FB00,FB06
 FB13,FB17
@@ -318,10 +312,7 @@ FB38,FB3C
 FB3E,FB3E
 FB40,FB41
 FB43,FB44
-FB46,FBC2
-FBD3,FD8F
-FD92,FDC7
-FDCF,FDCF
+FB46,FDCF
 FDF0,FE19
 FE20,FE52
 FE54,FE66
@@ -373,6 +364,7 @@ FFF9,FFFD
 105A3,105B1
 105B3,105B9
 105BB,105BC
+105C0,105F3
 10600,10736
 10740,10755
 10760,10767
@@ -391,7 +383,7 @@ FFF9,FFFD
 108F4,108F5
 108FB,1091B
 1091F,10939
-1093F,1093F
+1093F,10959
 10980,109B7
 109BC,109CF
 109D2,10A03
@@ -416,11 +408,16 @@ FFF9,FFFD
 10CC0,10CF2
 10CFA,10D27
 10D30,10D39
+10D40,10D65
+10D69,10D85
+10D8E,10D8F
 10E60,10E7E
 10E80,10EA9
 10EAB,10EAD
 10EB0,10EB1
-10F00,10F27
+10EC2,10EC7
+10ED0,10ED8
+10EFA,10F27
 10F30,10F59
 10F70,10F89
 10FB0,10FCB
@@ -437,7 +434,7 @@ FFF9,FFFD
 11180,111DF
 111E1,111F4
 11200,11211
-11213,1123E
+11213,11241
 11280,11286
 11288,11288
 1128A,1128D
@@ -460,6 +457,17 @@ FFF9,FFFD
 1135D,11363
 11366,1136C
 11370,11374
+11380,11389
+1138B,1138B
+1138E,1138E
+11390,113B5
+113B7,113C0
+113C2,113C2
+113C5,113C5
+113C7,113CA
+113CC,113D5
+113D7,113D8
+113E1,113E2
 11400,1145B
 1145D,11461
 11480,114C7
@@ -471,6 +479,7 @@ FFF9,FFFD
 11660,1166C
 11680,116B9
 116C0,116C9
+116D0,116E3
 11700,1171A
 1171D,1172B
 11730,11746
@@ -490,6 +499,10 @@ FFF9,FFFD
 11A00,11A47
 11A50,11AA2
 11AB0,11AF8
+11B00,11B09
+11B60,11B67
+11BC0,11BE1
+11BF0,11BF9
 11C00,11C08
 11C0A,11C36
 11C38,11C45
@@ -510,7 +523,12 @@ FFF9,FFFD
 11D90,11D91
 11D93,11D98
 11DA0,11DA9
+11DB0,11DDB
+11DE0,11DE9
 11EE0,11EF8
+11F00,11F10
+11F12,11F3A
+11F3E,11F5A
 11FB0,11FB0
 11FC0,11FF1
 11FFF,12399
@@ -518,9 +536,10 @@ FFF9,FFFD
 12470,12474
 12480,12543
 12F90,12FF2
-13000,1342E
-13430,13438
+13000,13455
+13460,143FA
 14400,14646
+16100,16139
 16800,16A38
 16A40,16A5E
 16A60,16A69
@@ -533,20 +552,25 @@ FFF9,FFFD
 16B5B,16B61
 16B63,16B77
 16B7D,16B8F
+16D40,16D79
 16E40,16E9A
+16EA0,16EB8
+16EBB,16ED3
 16F00,16F4A
 16F4F,16F87
 16F8F,16F9F
 16FE0,16FE4
-16FF0,16FF1
-17000,187F7
-18800,18CD5
-18D00,18D08
+16FF0,16FF6
+17000,18CD5
+18CFF,18D1E
+18D80,18DF2
 1AFF0,1AFF3
 1AFF5,1AFFB
 1AFFD,1AFFE
 1B000,1B122
+1B132,1B132
 1B150,1B152
+1B155,1B155
 1B164,1B167
 1B170,1B2FB
 1BC00,1BC6A
@@ -554,6 +578,10 @@ FFF9,FFFD
 1BC80,1BC88
 1BC90,1BC99
 1BC9C,1BCA3
+1CC00,1CCFC
+1CD00,1CEB3
+1CEBA,1CED0
+1CEE0,1CEF0
 1CF00,1CF2D
 1CF30,1CF46
 1CF50,1CFC3
@@ -561,6 +589,7 @@ FFF9,FFFD
 1D100,1D126
 1D129,1D1EA
 1D200,1D245
+1D2C0,1D2D3
 1D2E0,1D2F3
 1D300,1D356
 1D360,1D378
@@ -588,11 +617,14 @@ FFF9,FFFD
 1DA9B,1DA9F
 1DAA1,1DAAF
 1DF00,1DF1E
+1DF25,1DF2A
 1E000,1E006
 1E008,1E018
 1E01B,1E021
 1E023,1E024
 1E026,1E02A
+1E030,1E06D
+1E08F,1E08F
 1E100,1E12C
 1E130,1E13D
 1E140,1E149
@@ -600,6 +632,12 @@ FFF9,FFFD
 1E290,1E2AE
 1E2C0,1E2F9
 1E2FF,1E2FF
+1E4D0,1E4F9
+1E5D0,1E5FA
+1E5FF,1E5FF
+1E6C0,1E6DE
+1E6E0,1E6F5
+1E6FE,1E6FF
 1E7E0,1E7E6
 1E7E8,1E7EB
 1E7ED,1E7EE
@@ -657,11 +695,10 @@ FFF9,FFFD
 1F240,1F248
 1F250,1F251
 1F260,1F265
-1F300,1F6D7
-1F6DD,1F6EC
+1F300,1F6D8
+1F6DC,1F6EC
 1F6F0,1F6FC
-1F700,1F773
-1F780,1F7D8
+1F700,1F7D9
 1F7E0,1F7EB
 1F7F0,1F7F0
 1F800,1F80B
@@ -669,28 +706,28 @@ FFF9,FFFD
 1F850,1F859
 1F860,1F887
 1F890,1F8AD
-1F8B0,1F8B1
-1F900,1FA53
+1F8B0,1F8BB
+1F8C0,1F8C1
+1F8D0,1F8D8
+1F900,1FA57
 1FA60,1FA6D
-1FA70,1FA74
-1FA78,1FA7C
-1FA80,1FA86
-1FA90,1FAAC
-1FAB0,1FABA
-1FAC0,1FAC5
-1FAD0,1FAD9
-1FAE0,1FAE7
-1FAF0,1FAF6
+1FA70,1FA7C
+1FA80,1FA8A
+1FA8E,1FAC6
+1FAC8,1FAC8
+1FACD,1FADC
+1FADF,1FAEA
+1FAEF,1FAF8
 1FB00,1FB92
-1FB94,1FBCA
-1FBF0,1FBF9
+1FB94,1FBFA
 20000,2A6DF
-2A700,2B738
-2B740,2B81D
-2B820,2CEA1
+2A700,2B81D
+2B820,2CEAD
 2CEB0,2EBE0
+2EBF0,2EE5D
 2F800,2FA1D
 30000,3134A
+31350,33479
 E0001,E0001
 E0020,E007F
 E0100,E01EF

data/lib/character_set/predefined_sets/emoji.cps

@@ -131,8 +131,8 @@ AE,AE
 1F5FA,1F64F
 1F680,1F6C5
 1F6CB,1F6D2
-1F6D5,1F6D7
-1F6DD,1F6E5
+1F6D5,1F6D8
+1F6DC,1F6E5
 1F6E9,1F6E9
 1F6EB,1F6EC
 1F6F0,1F6F0
@@ -142,12 +142,10 @@ AE,AE
 1F90C,1F93A
 1F93C,1F945
 1F947,1F9FF
-1FA70,1FA74
-1FA78,1FA7C
-1FA80,1FA86
-1FA90,1FAAC
-1FAB0,1FABA
-1FAC0,1FAC5
-1FAD0,1FAD9
-1FAE0,1FAE7
-1FAF0,1FAF6
+1FA70,1FA7C
+1FA80,1FA8A
+1FA8E,1FAC6
+1FAC8,1FAC8
+1FACD,1FADC
+1FADF,1FAEA
+1FAEF,1FAF8

data/lib/character_set/predefined_sets.rb

@@ -23,7 +23,7 @@ class CharacterSet
 
     def build_from_cps_file(path)
       if defined?(Ractor) && Ractor.current != Ractor.main
-        raise <<-EOS.gsub(/^ */, '')
+        raise <<~EOS
           CharacterSet's predefined sets are lazy-loaded.
           Pre-load them to use them in Ractors. E.g.:
 

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -44,12 +44,11 @@ class CharacterSet
       end
 
       def cover?(string)
-        utf8_str!(string).each_codepoint { |cp| return false unless include?(cp) }
-        true
+        utf8_str!(string).each_codepoint.all? { |cp| include?(cp) }
       end
 
       def delete_in(string)
-        utf8_str!(string).each_codepoint.with_object('') do |cp, new_str|
+        utf8_str!(string).each_codepoint.with_object(+'') do |cp, new_str|
           include?(cp) || (new_str << cp)
         end.encode(string.encoding)
       end
@@ -60,7 +59,7 @@ class CharacterSet
       end
 
       def keep_in(string)
-        utf8_str!(string).each_codepoint.with_object('') do |cp, new_str|
+        utf8_str!(string).each_codepoint.with_object(+'') do |cp, new_str|
           include?(cp) && (new_str << cp)
         end.encode(string.encoding)
       end
@@ -71,14 +70,13 @@ class CharacterSet
       end
 
       def scan(string)
-        utf8_str!(string).each_codepoint.with_object([]) do |cp, arr|
-          arr.push(cp.chr('utf-8')) if include?(cp)
+        utf8_str!(string).each_codepoint.filter_map do |cp|
+          cp.chr('utf-8') if include?(cp)
         end
       end
 
       def used_by?(string)
-        utf8_str!(string).each_codepoint { |cp| return true if include?(cp) }
-        false
+        utf8_str!(string).each_codepoint.any? { |cp| include?(cp) }
       end
 
       def section(from:, upto: 0x10FFFF)
@@ -98,8 +96,7 @@ class CharacterSet
       end
 
       def planes
-        plane_size = 0x10000.to_f
-        inject({}) { |hash, cp| hash.merge((cp / plane_size).floor => 1) }.keys
+        map { |cp| cp / 0x10000 }.uniq
       end
 
       def plane(num)
@@ -122,10 +119,6 @@ class CharacterSet
         raise ArgumentError, 'pass a String' unless obj.respond_to?(:codepoints)
         obj.encode('utf-8')
       end
-
-      def make_new_str(original, &block)
-        utf8_str!(original).each_codepoint.with_object('', &block)
-      end
     end
   end
 end