character_set 1.2.0 → 1.5.0

data/ext/character_set/unicode_casefold_table.h

@@ -6,7 +6,7 @@ typedef struct casefold_mapping {
   unsigned long to;
 } casefold_mapping;
 
-#define CASEFOLD_COUNT 1383
+#define CASEFOLD_COUNT 1426
 
 static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x0041,0x0061},
@@ -564,6 +564,41 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x104D1,0x104F9},
 {0x104D2,0x104FA},
 {0x104D3,0x104FB},
+{0x10570,0x10597},
+{0x10571,0x10598},
+{0x10572,0x10599},
+{0x10573,0x1059A},
+{0x10574,0x1059B},
+{0x10575,0x1059C},
+{0x10576,0x1059D},
+{0x10577,0x1059E},
+{0x10578,0x1059F},
+{0x10579,0x105A0},
+{0x1057A,0x105A1},
+{0x1057C,0x105A3},
+{0x1057D,0x105A4},
+{0x1057E,0x105A5},
+{0x1057F,0x105A6},
+{0x10580,0x105A7},
+{0x10581,0x105A8},
+{0x10582,0x105A9},
+{0x10583,0x105AA},
+{0x10584,0x105AB},
+{0x10585,0x105AC},
+{0x10586,0x105AD},
+{0x10587,0x105AE},
+{0x10588,0x105AF},
+{0x10589,0x105B0},
+{0x1058A,0x105B1},
+{0x1058C,0x105B3},
+{0x1058D,0x105B4},
+{0x1058E,0x105B5},
+{0x1058F,0x105B6},
+{0x10590,0x105B7},
+{0x10591,0x105B8},
+{0x10592,0x105B9},
+{0x10594,0x105BB},
+{0x10595,0x105BC},
 {0x10A0,0x2D00},
 {0x10A1,0x2D01},
 {0x10A2,0x2D02},
@@ -1102,6 +1137,7 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x2C2C,0x2C5C},
 {0x2C2D,0x2C5D},
 {0x2C2E,0x2C5E},
+{0x2C2F,0x2C5F},
 {0x2C60,0x2C61},
 {0x2C62,0x026B},
 {0x2C63,0x1D7D},
@@ -1282,10 +1318,17 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0xA7BA,0xA7BB},
 {0xA7BC,0xA7BD},
 {0xA7BE,0xA7BF},
+{0xA7C0,0xA7C1},
 {0xA7C2,0xA7C3},
 {0xA7C4,0xA794},
 {0xA7C5,0x0282},
 {0xA7C6,0x1D8E},
+{0xA7C7,0xA7C8},
+{0xA7C9,0xA7CA},
+{0xA7D0,0xA7D1},
+{0xA7D6,0xA7D7},
+{0xA7D8,0xA7D9},
+{0xA7F5,0xA7F6},
 {0xAB70,0x13A0},
 {0xAB71,0x13A1},
 {0xAB72,0x13A2},

data/ext/character_set/unicode_casefold_table.h.tmpl

@@ -0,0 +1,11 @@
+// THIS FILE IS GENERATED BY $ rake sync_casefold_data - DO NOT EDIT
+// -*-C-*-
+
+typedef struct casefold_mapping {
+  unsigned long from;
+  unsigned long to;
+} casefold_mapping;
+
+#define CASEFOLD_COUNT 0
+
+static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {};

data/lib/character_set/character.rb

@@ -1,7 +1,7 @@
 class CharacterSet
   class Character
     ENCODING = 'utf-8'.freeze
-    SAFELY_PRINTABLE = (0x21..0x7E).to_a - ['-', '[', '\\', ']', '^'].map(&:ord)
+    SAFELY_PRINTABLE = (0x21..0x7E).to_a - %w(- / [ \\ ] ^).map(&:ord)
 
     attr_accessor :codepoint
 

data/lib/character_set/core_ext/regexp_ext.rb

@@ -8,4 +8,4 @@ class CharacterSet
   end
 end
 
-::Regexp.send(:include, CharacterSet::CoreExt::RegexpExt)
+::Regexp.instance_eval { include CharacterSet::CoreExt::RegexpExt }

data/lib/character_set/core_ext/string_ext.rb

@@ -6,11 +6,13 @@ class CharacterSet
       end
 
       {
+        count_by_character_set:    :count_in,
         covered_by_character_set?: :cover?,
         delete_character_set:      :delete_in,
         delete_character_set!:     :delete_in!,
         keep_character_set:        :keep_in,
         keep_character_set!:       :keep_in!,
+        scan_by_character_set:     :scan,
         uses_character_set?:       :used_by?,
       }.each do |string_method, set_method|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
@@ -27,4 +29,4 @@ class CharacterSet
   end
 end
 
-::String.send(:include, CharacterSet::CoreExt::StringExt)
+::String.instance_eval { include CharacterSet::CoreExt::StringExt }

data/lib/character_set/expression_converter.rb

@@ -4,62 +4,57 @@ class CharacterSet
 
     Error = Class.new(ArgumentError)
 
-    def convert(expression)
-      CharacterSet.require_optional_dependency('regexp_parser')
+    def convert(expression, to = CharacterSet)
+      CharacterSet.require_optional_dependency('regexp_parser', __method__)
 
       case expression
       when Regexp::Expression::Root
         if expression.count != 1
           raise Error, 'Pass a Regexp with exactly one expression, e.g. /[a-z]/'
         end
-        convert(expression[0])
+        convert(expression[0], to)
 
       when Regexp::Expression::CharacterSet
-        content = expression.map { |subexp| convert(subexp) }.reduce(:+)
+        content = expression.map { |subexp| convert(subexp, to) }.reduce(:+)
+        content ||= to[]
         expression.negative? ? content.inversion : content
 
       when Regexp::Expression::CharacterSet::Intersection
-        expression.map { |subexp| convert(subexp) }.reduce(:&)
+        expression.map { |subexp| convert(subexp, to) }.reduce(:&)
 
       when Regexp::Expression::CharacterSet::IntersectedSequence
-        expression.map { |subexp| convert(subexp) }.reduce(:+)
+        expression.map { |subexp| convert(subexp, to) }.reduce(:+) || to[]
 
       when Regexp::Expression::CharacterSet::Range
-        start, finish = expression.map { |subexp| convert(subexp) }
-        CharacterSet.from_ranges((start.min)..(finish.max))
+        start, finish = expression.map { |subexp| convert(subexp, to) }
+        to.new((start.min)..(finish.max))
 
       when Regexp::Expression::CharacterType::Any
-        CharacterSet.unicode
-
-      when Regexp::Expression::CharacterType::Digit
-        CharacterSet.from_ranges(48..57)
-
-      when Regexp::Expression::CharacterType::NonDigit
-        CharacterSet.from_ranges(48..57).inversion
-
-      when Regexp::Expression::CharacterType::Hex
-        CharacterSet.from_ranges(48..57, 65..70, 97..102)
-
-      when Regexp::Expression::CharacterType::NonHex
-        CharacterSet.from_ranges(48..57, 65..70, 97..102).inversion
-
-      when Regexp::Expression::CharacterType::Space
-        CharacterSet["\t", "\n", "\v", "\f", "\r", "\x20"]
-
-      when Regexp::Expression::CharacterType::NonSpace
-        CharacterSet["\t", "\n", "\v", "\f", "\r", "\x20"].inversion
-
-      when Regexp::Expression::CharacterType::Word
-        CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122)
-
-      when Regexp::Expression::CharacterType::NonWord
-        CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122).inversion
+        to.unicode
+
+      when Regexp::Expression::CharacterType::Base
+        /(?<negative>non)?(?<base_name>.+)/ =~ expression.token
+        content =
+          if expression.unicode_classes?
+            # in u-mode, type shortcuts match the same as \p{<long type name>}
+            to.of_property(base_name)
+          else
+            # in normal mode, types match only ascii chars
+            case base_name.to_sym
+            when :digit then to.from_ranges(48..57)
+            when :hex   then to.from_ranges(48..57, 65..70, 97..102)
+            when :space then to.from_ranges(9..13, 32..32)
+            when :word  then to.from_ranges(48..57, 65..90, 95..95, 97..122)
+            else raise Error, "Unsupported CharacterType #{base_name}"
+            end
+          end
+        negative ? content.inversion : content
 
       when Regexp::Expression::EscapeSequence::CodepointList
-        CharacterSet.new(expression.codepoints)
+        to.new(expression.codepoints)
 
       when Regexp::Expression::EscapeSequence::Base
-        CharacterSet[expression.codepoint]
+        to[expression.codepoint]
 
       when Regexp::Expression::Group::Capture,
            Regexp::Expression::Group::Passive,
@@ -67,19 +62,19 @@ class CharacterSet
            Regexp::Expression::Group::Atomic,
            Regexp::Expression::Group::Options
         case expression.count
-        when 0 then CharacterSet[]
-        when 1 then convert(expression.first)
+        when 0 then to[]
+        when 1 then convert(expression.first, to)
         else
           raise Error, 'Groups must contain exactly one expression, e.g. ([a-z])'
         end
 
-      when Regexp::Expression::Alternation
-        expression.map { |subexp| convert(subexp) }.reduce(:+)
+      when Regexp::Expression::Alternation # rubocop:disable Lint/DuplicateBranch
+        expression.map { |subexp| convert(subexp, to) }.reduce(:+)
 
       when Regexp::Expression::Alternative
         case expression.count
-        when 0 then CharacterSet[]
-        when 1 then convert(expression.first)
+        when 0 then to[]
+        when 1 then convert(expression.first, to)
         else
           raise Error, 'Alternatives must contain exactly one expression'
         end
@@ -88,11 +83,14 @@ class CharacterSet
         if expression.set_level == 0 && expression.text.size != 1
           raise Error, 'Literal runs outside of sets are codepoint *sequences*'
         end
-        CharacterSet[expression.text.ord]
+        to[expression.text.ord]
 
       when Regexp::Expression::UnicodeProperty::Base,
            Regexp::Expression::PosixClass
-        content = CharacterSet.of_property(expression.token)
+        content = to.of_property(expression.token)
+        if expression.type == :posixclass && expression.ascii_classes?
+          content = content.ascii_part
+        end
         expression.negative? ? content.inversion : content
 
       when Regexp::Expression::Base

data/lib/character_set/parser.rb

@@ -5,7 +5,7 @@ class CharacterSet
     def codepoints_from_enumerable(object)
       raise ArgumentError, 'pass an Enumerable' unless object.respond_to?(:each)
       # Use #each to check first element (only this works for all Enumerables)
-      object.each do |e|
+      object.each do |e| # rubocop:disable Lint/UnreachableLoop
         return object            if e.is_a?(Integer) && e >= 0 && e < 0x110000
         return object.map(&:ord) if e.is_a?(String)  && e.length == 1
         raise ArgumentError, "#{e.inspect} is not valid as a codepoint"

data/lib/character_set/predefined_sets/any.cps

@@ -0,0 +1 @@
+0,10FFFF

data/lib/character_set/predefined_sets/ascii.cps

@@ -0,0 +1 @@
+0,7F

data/lib/character_set/predefined_sets/ascii_alnum.cps

@@ -0,0 +1,3 @@
+30,39
+41,5A
+61,7A

data/lib/character_set/predefined_sets/ascii_letter.cps

@@ -0,0 +1,2 @@
+41,5A
+61,7A