character_set 1.1.1 → 1.4.1

data/ext/character_set/unicode_casefold_table.h

@@ -1,11 +1,12 @@
-// THIS FILE IS GENERATED BY $ rake sync_casefold_data - DO NOT EDIT'
+// THIS FILE IS GENERATED BY $ rake sync_casefold_data - DO NOT EDIT
+// -*-C-*-
 
 typedef struct casefold_mapping {
   unsigned long from;
   unsigned long to;
 } casefold_mapping;
 
-#define CASEFOLD_COUNT 1376
+#define CASEFOLD_COUNT 1383
 
 static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x0041,0x0061},
@@ -1278,6 +1279,13 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0xA7B4,0xA7B5},
 {0xA7B6,0xA7B7},
 {0xA7B8,0xA7B9},
+{0xA7BA,0xA7BB},
+{0xA7BC,0xA7BD},
+{0xA7BE,0xA7BF},
+{0xA7C2,0xA7C3},
+{0xA7C4,0xA794},
+{0xA7C5,0x0282},
+{0xA7C6,0x1D8E},
 {0xAB70,0x13A0},
 {0xAB71,0x13A1},
 {0xAB72,0x13A2},

data/ext/character_set/unicode_casefold_table.h.tmpl

@@ -0,0 +1,11 @@
+// THIS FILE IS GENERATED BY $ rake sync_casefold_data - DO NOT EDIT
+// -*-C-*-
+
+typedef struct casefold_mapping {
+  unsigned long from;
+  unsigned long to;
+} casefold_mapping;
+
+#define CASEFOLD_COUNT 0
+
+static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {};

data/lib/character_set/character.rb

@@ -1,7 +1,7 @@
 class CharacterSet
   class Character
     ENCODING = 'utf-8'.freeze
-    SAFELY_PRINTABLE = (0x21..0x7E).to_a - ['-', '[', '\\', ']', '^'].map(&:ord)
+    SAFELY_PRINTABLE = (0x21..0x7E).to_a - %w(- / [ \\ ] ^).map(&:ord)
 
     attr_accessor :codepoint
 

data/lib/character_set/core_ext/regexp_ext.rb

@@ -8,4 +8,4 @@ class CharacterSet
   end
 end
 
-::Regexp.send(:include, CharacterSet::CoreExt::RegexpExt)
+::Regexp.instance_eval { include CharacterSet::CoreExt::RegexpExt }

data/lib/character_set/core_ext/string_ext.rb

@@ -6,11 +6,13 @@ class CharacterSet
       end
 
       {
+        count_by_character_set:    :count_in,
         covered_by_character_set?: :cover?,
         delete_character_set:      :delete_in,
         delete_character_set!:     :delete_in!,
         keep_character_set:        :keep_in,
         keep_character_set!:       :keep_in!,
+        scan_by_character_set:     :scan,
         uses_character_set?:       :used_by?,
       }.each do |string_method, set_method|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
@@ -27,4 +29,4 @@ class CharacterSet
   end
 end
 
-::String.send(:include, CharacterSet::CoreExt::StringExt)
+::String.instance_eval { include CharacterSet::CoreExt::StringExt }

data/lib/character_set/expression_converter.rb

@@ -5,7 +5,7 @@ class CharacterSet
     Error = Class.new(ArgumentError)
 
     def convert(expression)
-      CharacterSet.require_optional_dependency('regexp_parser')
+      CharacterSet.require_optional_dependency('regexp_parser', __method__)
 
       case expression
       when Regexp::Expression::Root
@@ -16,44 +16,39 @@ class CharacterSet
 
       when Regexp::Expression::CharacterSet
         content = expression.map { |subexp| convert(subexp) }.reduce(:+)
+        content ||= CharacterSet[]
         expression.negative? ? content.inversion : content
 
       when Regexp::Expression::CharacterSet::Intersection
         expression.map { |subexp| convert(subexp) }.reduce(:&)
 
       when Regexp::Expression::CharacterSet::IntersectedSequence
-        expression.map { |subexp| convert(subexp) }.reduce(:+)
+        expression.map { |subexp| convert(subexp) }.reduce(:+) || CharacterSet[]
 
       when Regexp::Expression::CharacterSet::Range
         start, finish = expression.map { |subexp| convert(subexp) }
-        CharacterSet.from_ranges((start.min)..(finish.max))
+        CharacterSet.new((start.min)..(finish.max))
 
       when Regexp::Expression::CharacterType::Any
         CharacterSet.unicode
 
-      when Regexp::Expression::CharacterType::Digit
-        CharacterSet.from_ranges(48..57)
-
-      when Regexp::Expression::CharacterType::NonDigit
-        CharacterSet.from_ranges(48..57).inversion
-
-      when Regexp::Expression::CharacterType::Hex
-        CharacterSet.from_ranges(48..57, 65..70, 97..102)
-
-      when Regexp::Expression::CharacterType::NonHex
-        CharacterSet.from_ranges(48..57, 65..70, 97..102).inversion
-
-      when Regexp::Expression::CharacterType::Space
-        CharacterSet["\t", "\n", "\v", "\f", "\r", "\x20"]
-
-      when Regexp::Expression::CharacterType::NonSpace
-        CharacterSet["\t", "\n", "\v", "\f", "\r", "\x20"].inversion
-
-      when Regexp::Expression::CharacterType::Word
-        CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122)
-
-      when Regexp::Expression::CharacterType::NonWord
-        CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122).inversion
+      when Regexp::Expression::CharacterType::Base
+        /(?<negative>non)?(?<base_name>.+)/ =~ expression.token
+        content =
+          if expression.unicode_classes?
+            # in u-mode, type shortcuts match the same as \p{<long type name>}
+            CharacterSet.of_property(base_name)
+          else
+            # in normal mode, types match only ascii chars
+            case base_name.to_sym
+            when :digit then CharacterSet.from_ranges(48..57)
+            when :hex   then CharacterSet.from_ranges(48..57, 65..70, 97..102)
+            when :space then CharacterSet.from_ranges(9..13, 32..32)
+            when :word  then CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122)
+            else raise Error, "Unsupported CharacterType #{base_name}"
+            end
+          end
+        negative ? content.inversion : content
 
       when Regexp::Expression::EscapeSequence::CodepointList
         CharacterSet.new(expression.codepoints)
@@ -73,7 +68,7 @@ class CharacterSet
           raise Error, 'Groups must contain exactly one expression, e.g. ([a-z])'
         end
 
-      when Regexp::Expression::Alternation
+      when Regexp::Expression::Alternation # rubocop:disable Lint/DuplicateBranch
         expression.map { |subexp| convert(subexp) }.reduce(:+)
 
       when Regexp::Expression::Alternative
@@ -93,6 +88,9 @@ class CharacterSet
       when Regexp::Expression::UnicodeProperty::Base,
            Regexp::Expression::PosixClass
         content = CharacterSet.of_property(expression.token)
+        if expression.type == :posixclass && expression.ascii_classes?
+          content = content.ascii_part
+        end
         expression.negative? ? content.inversion : content
 
       when Regexp::Expression::Base

data/lib/character_set/parser.rb

@@ -5,7 +5,7 @@ class CharacterSet
     def codepoints_from_enumerable(object)
       raise ArgumentError, 'pass an Enumerable' unless object.respond_to?(:each)
       # Use #each to check first element (only this works for all Enumerables)
-      object.each do |e|
+      object.each do |e| # rubocop:disable Lint/UnreachableLoop
         return object            if e.is_a?(Integer) && e >= 0 && e < 0x110000
         return object.map(&:ord) if e.is_a?(String)  && e.length == 1
         raise ArgumentError, "#{e.inspect} is not valid as a codepoint"

data/lib/character_set/predefined_sets.rb

@@ -1,266 +1,31 @@
 class CharacterSet
   module PredefinedSets
-    def ascii
-      @ascii ||= from_ranges(0..0x7F).freeze
-    end
-
-    def ascii_alnum
-      @ascii_alnum ||= from_ranges(0x30..0x39, 0x41..0x5A, 0x61..0x7A).freeze
-    end
-
-    def ascii_letters
-      @ascii_letters ||= from_ranges(0x41..0x5A, 0x61..0x7A).freeze
-    end
-
-    # basic multilingual plane
-    def bmp
-      @bmp ||= from_ranges(0..0xD7FF, 0xE000..0xFFFF).freeze
-    end
-
-    # ./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
-    def crypt
-      @crypt ||= from_ranges(0x2E..0x5A, 0x61..0x7A).freeze
-    end
-
-    def newline
-      @newline ||= from_ranges(0xA..0xD, 0x85..0x85, 0x2028..0x2029).freeze
-    end
-
-    def unicode
-      @unicode ||= from_ranges(0..0xD7FF, 0xE000..0x10FFFF).freeze
-    end
-
-    def url_fragment
-      @url_fragment ||= from_ranges(
-        0x21..0x21,
-        0x24..0x24,
-        0x26..0x3B,
-        0x3D..0x3D,
-        0x3F..0x5A,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def url_host
-      @url_host ||= from_ranges(
-        0x21..0x21,
-        0x24..0x24,
-        0x26..0x2E,
-        0x30..0x3B,
-        0x3D..0x3D,
-        0x41..0x5B,
-        0x5D..0x5D,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def url_path
-      @url_path ||= from_ranges(
-        0x21..0x21,
-        0x24..0x3A,
-        0x3D..0x3D,
-        0x40..0x5A,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def url_query
-      @url_query ||= from_ranges(
-        0x21..0x21,
-        0x24..0x24,
-        0x26..0x3B,
-        0x3D..0x3D,
-        0x3F..0x5A,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def whitespace
-      @whitespace ||= from_ranges(
-        0x9..0x9,
-        0xA..0xD,
-        0x20..0x20,
-        0x85..0x85,
-        0xA0..0xA0,
-        0x1680..0x1680,
-        0x180E..0x180E,
-        0x2000..0x200A,
-        0x2028..0x2029,
-        0x202F..0x202F,
-        0x205F..0x205F,
-        0x3000..0x3000
-      ).freeze
-    end
-
-    def emoji
-      @emoji ||= from_ranges(
-        0x23..0x23,
-        0x2A..0x2A,
-        0x30..0x39,
-        0xA9..0xA9,
-        0xAE..0xAE,
-        0x203C..0x203C,
-        0x2049..0x2049,
-        0x2122..0x2122,
-        0x2139..0x2139,
-        0x2194..0x2199,
-        0x21A9..0x21AA,
-        0x231A..0x231B,
-        0x2328..0x2328,
-        0x23CF..0x23CF,
-        0x23E9..0x23F3,
-        0x23F8..0x23FA,
-        0x24C2..0x24C2,
-        0x25AA..0x25AB,
-        0x25B6..0x25B6,
-        0x25C0..0x25C0,
-        0x25FB..0x25FE,
-        0x2600..0x2604,
-        0x260E..0x260E,
-        0x2611..0x2611,
-        0x2614..0x2615,
-        0x2618..0x2618,
-        0x261D..0x261D,
-        0x2620..0x2620,
-        0x2622..0x2623,
-        0x2626..0x2626,
-        0x262A..0x262A,
-        0x262E..0x262F,
-        0x2638..0x263A,
-        0x2640..0x2640,
-        0x2642..0x2642,
-        0x2648..0x2653,
-        0x2660..0x2660,
-        0x2663..0x2663,
-        0x2665..0x2666,
-        0x2668..0x2668,
-        0x267B..0x267B,
-        0x267F..0x267F,
-        0x2692..0x2697,
-        0x2699..0x2699,
-        0x269B..0x269C,
-        0x26A0..0x26A1,
-        0x26AA..0x26AB,
-        0x26B0..0x26B1,
-        0x26BD..0x26BE,
-        0x26C4..0x26C5,
-        0x26C8..0x26C8,
-        0x26CE..0x26CF,
-        0x26D1..0x26D1,
-        0x26D3..0x26D4,
-        0x26E9..0x26EA,
-        0x26F0..0x26F5,
-        0x26F7..0x26FA,
-        0x26FD..0x26FD,
-        0x2702..0x2702,
-        0x2705..0x2705,
-        0x2708..0x270D,
-        0x270F..0x270F,
-        0x2712..0x2712,
-        0x2714..0x2714,
-        0x2716..0x2716,
-        0x271D..0x271D,
-        0x2721..0x2721,
-        0x2728..0x2728,
-        0x2733..0x2734,
-        0x2744..0x2744,
-        0x2747..0x2747,
-        0x274C..0x274C,
-        0x274E..0x274E,
-        0x2753..0x2755,
-        0x2757..0x2757,
-        0x2763..0x2764,
-        0x2795..0x2797,
-        0x27A1..0x27A1,
-        0x27B0..0x27B0,
-        0x27BF..0x27BF,
-        0x2934..0x2935,
-        0x2B05..0x2B07,
-        0x2B1B..0x2B1C,
-        0x2B50..0x2B50,
-        0x2B55..0x2B55,
-        0x3030..0x3030,
-        0x303D..0x303D,
-        0x3297..0x3297,
-        0x3299..0x3299,
-        0x1F004..0x1F004,
-        0x1F0CF..0x1F0CF,
-        0x1F170..0x1F171,
-        0x1F17E..0x1F17F,
-        0x1F18E..0x1F18E,
-        0x1F191..0x1F19A,
-        0x1F1E6..0x1F1FF,
-        0x1F201..0x1F202,
-        0x1F21A..0x1F21A,
-        0x1F22F..0x1F22F,
-        0x1F232..0x1F23A,
-        0x1F250..0x1F251,
-        0x1F300..0x1F321,
-        0x1F324..0x1F393,
-        0x1F396..0x1F397,
-        0x1F399..0x1F39B,
-        0x1F39E..0x1F3F0,
-        0x1F3F3..0x1F3F5,
-        0x1F3F7..0x1F4FD,
-        0x1F4FF..0x1F53D,
-        0x1F549..0x1F54E,
-        0x1F550..0x1F567,
-        0x1F56F..0x1F570,
-        0x1F573..0x1F57A,
-        0x1F587..0x1F587,
-        0x1F58A..0x1F58D,
-        0x1F590..0x1F590,
-        0x1F595..0x1F596,
-        0x1F5A4..0x1F5A5,
-        0x1F5A8..0x1F5A8,
-        0x1F5B1..0x1F5B2,
-        0x1F5BC..0x1F5BC,
-        0x1F5C2..0x1F5C4,
-        0x1F5D1..0x1F5D3,
-        0x1F5DC..0x1F5DE,
-        0x1F5E1..0x1F5E1,
-        0x1F5E3..0x1F5E3,
-        0x1F5E8..0x1F5E8,
-        0x1F5EF..0x1F5EF,
-        0x1F5F3..0x1F5F3,
-        0x1F5FA..0x1F64F,
-        0x1F680..0x1F6C5,
-        0x1F6CB..0x1F6D2,
-        0x1F6E0..0x1F6E5,
-        0x1F6E9..0x1F6E9,
-        0x1F6EB..0x1F6EC,
-        0x1F6F0..0x1F6F0,
-        0x1F6F3..0x1F6F8,
-        0x1F910..0x1F93A,
-        0x1F93C..0x1F93E,
-        0x1F940..0x1F945,
-        0x1F947..0x1F94C,
-        0x1F950..0x1F96B,
-        0x1F980..0x1F997,
-        0x1F9C0..0x1F9C0,
-        0x1F9D0..0x1F9E6
-      ).freeze
-    end
-
-    def respond_to_missing?(method_name, include_private = false)
-      (base = method_name[/^non_(.*)/, 1]) && respond_to?(base) || super
-    end
-
-    def method_missing(method_name, *args, &block)
-      if (base = method_name[/^non_(.*)/, 1])
-        ivar_name = "@#{method_name}"
-        return instance_variable_get(ivar_name) ||
-               instance_variable_set(ivar_name, send(base).inversion.freeze)
+    Dir[File.join(__dir__, 'predefined_sets', '*.cps')].each do |path|
+      set_name = File.basename(path, '.cps')
+
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
+        def #{set_name}
+          @#{set_name} ||= build_from_cps_file('#{path}').freeze
+        end
+
+        def non_#{set_name}
+          @non_#{set_name} ||= build_from_cps_file('#{path}').inversion.freeze
+        end
+      RUBY
+    end
+
+    alias all                      any
+    alias ascii_letters            ascii_letter
+    alias basic_multilingual_plane bmp
+    alias blank                    whitespace
+    alias invalid                  surrogate
+    alias valid                    unicode
+
+    def build_from_cps_file(path)
+      File.readlines(path).inject(new) do |set, line|
+        range_start, range_end = line.split(',')
+        set.merge((range_start.to_i(16))..(range_end.to_i(16)))
       end
-      super
     end
   end
 end