character_set 1.2.0-java → 1.5.0-java

data/lib/character_set/predefined_sets/emoji.cps

@@ -0,0 +1,152 @@
+23,23
+2A,2A
+30,39
+A9,A9
+AE,AE
+203C,203C
+2049,2049
+2122,2122
+2139,2139
+2194,2199
+21A9,21AA
+231A,231B
+2328,2328
+23CF,23CF
+23E9,23F3
+23F8,23FA
+24C2,24C2
+25AA,25AB
+25B6,25B6
+25C0,25C0
+25FB,25FE
+2600,2604
+260E,260E
+2611,2611
+2614,2615
+2618,2618
+261D,261D
+2620,2620
+2622,2623
+2626,2626
+262A,262A
+262E,262F
+2638,263A
+2640,2640
+2642,2642
+2648,2653
+265F,2660
+2663,2663
+2665,2666
+2668,2668
+267B,267B
+267E,267F
+2692,2697
+2699,2699
+269B,269C
+26A0,26A1
+26A7,26A7
+26AA,26AB
+26B0,26B1
+26BD,26BE
+26C4,26C5
+26C8,26C8
+26CE,26CF
+26D1,26D1
+26D3,26D4
+26E9,26EA
+26F0,26F5
+26F7,26FA
+26FD,26FD
+2702,2702
+2705,2705
+2708,270D
+270F,270F
+2712,2712
+2714,2714
+2716,2716
+271D,271D
+2721,2721
+2728,2728
+2733,2734
+2744,2744
+2747,2747
+274C,274C
+274E,274E
+2753,2755
+2757,2757
+2763,2764
+2795,2797
+27A1,27A1
+27B0,27B0
+27BF,27BF
+2934,2935
+2B05,2B07
+2B1B,2B1C
+2B50,2B50
+2B55,2B55
+3030,3030
+303D,303D
+3297,3297
+3299,3299
+1F004,1F004
+1F0CF,1F0CF
+1F170,1F171
+1F17E,1F17F
+1F18E,1F18E
+1F191,1F19A
+1F1E6,1F1FF
+1F201,1F202
+1F21A,1F21A
+1F22F,1F22F
+1F232,1F23A
+1F250,1F251
+1F300,1F321
+1F324,1F393
+1F396,1F397
+1F399,1F39B
+1F39E,1F3F0
+1F3F3,1F3F5
+1F3F7,1F4FD
+1F4FF,1F53D
+1F549,1F54E
+1F550,1F567
+1F56F,1F570
+1F573,1F57A
+1F587,1F587
+1F58A,1F58D
+1F590,1F590
+1F595,1F596
+1F5A4,1F5A5
+1F5A8,1F5A8
+1F5B1,1F5B2
+1F5BC,1F5BC
+1F5C2,1F5C4
+1F5D1,1F5D3
+1F5DC,1F5DE
+1F5E1,1F5E1
+1F5E3,1F5E3
+1F5E8,1F5E8
+1F5EF,1F5EF
+1F5F3,1F5F3
+1F5FA,1F64F
+1F680,1F6C5
+1F6CB,1F6D2
+1F6D5,1F6D7
+1F6E0,1F6E5
+1F6E9,1F6E9
+1F6EB,1F6EC
+1F6F0,1F6F0
+1F6F3,1F6FC
+1F7E0,1F7EB
+1F90C,1F93A
+1F93C,1F945
+1F947,1F978
+1F97A,1F9CB
+1F9CD,1F9FF
+1FA70,1FA74
+1FA78,1FA7A
+1FA80,1FA86
+1FA90,1FAA8
+1FAB0,1FAB6
+1FAC0,1FAC2
+1FAD0,1FAD6

data/lib/character_set/predefined_sets/newline.cps

@@ -0,0 +1,3 @@
+A,D
+85,85
+2028,2029

data/lib/character_set/predefined_sets/surrogate.cps

@@ -0,0 +1 @@
+D800,DFFF

data/lib/character_set/predefined_sets/unicode.cps

@@ -0,0 +1,2 @@
+0,D7FF
+E000,10FFFF

data/lib/character_set/predefined_sets/url_fragment.cps

@@ -0,0 +1,8 @@
+21,21
+24,24
+26,3B
+3D,3D
+3F,5A
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/url_host.cps

@@ -0,0 +1,10 @@
+21,21
+24,24
+26,2E
+30,3B
+3D,3D
+41,5B
+5D,5D
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/url_path.cps

@@ -0,0 +1,7 @@
+21,21
+24,3A
+3D,3D
+40,5A
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/url_query.cps

@@ -0,0 +1,8 @@
+21,21
+24,24
+26,3B
+3D,3D
+3F,5A
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/whitespace.cps

@@ -0,0 +1,10 @@
+9,D
+20,20
+85,85
+A0,A0
+1680,1680
+2000,200A
+2028,2029
+202F,202F
+205F,205F
+3000,3000

data/lib/character_set/predefined_sets.rb

@@ -1,266 +1,31 @@
 class CharacterSet
   module PredefinedSets
-    def ascii
-      @ascii ||= from_ranges(0..0x7F).freeze
-    end
-
-    def ascii_alnum
-      @ascii_alnum ||= from_ranges(0x30..0x39, 0x41..0x5A, 0x61..0x7A).freeze
-    end
-
-    def ascii_letters
-      @ascii_letters ||= from_ranges(0x41..0x5A, 0x61..0x7A).freeze
-    end
-
-    # basic multilingual plane
-    def bmp
-      @bmp ||= from_ranges(0..0xD7FF, 0xE000..0xFFFF).freeze
-    end
-
-    # ./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
-    def crypt
-      @crypt ||= from_ranges(0x2E..0x5A, 0x61..0x7A).freeze
-    end
-
-    def newline
-      @newline ||= from_ranges(0xA..0xD, 0x85..0x85, 0x2028..0x2029).freeze
-    end
-
-    def unicode
-      @unicode ||= from_ranges(0..0xD7FF, 0xE000..0x10FFFF).freeze
-    end
-
-    def url_fragment
-      @url_fragment ||= from_ranges(
-        0x21..0x21,
-        0x24..0x24,
-        0x26..0x3B,
-        0x3D..0x3D,
-        0x3F..0x5A,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def url_host
-      @url_host ||= from_ranges(
-        0x21..0x21,
-        0x24..0x24,
-        0x26..0x2E,
-        0x30..0x3B,
-        0x3D..0x3D,
-        0x41..0x5B,
-        0x5D..0x5D,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def url_path
-      @url_path ||= from_ranges(
-        0x21..0x21,
-        0x24..0x3A,
-        0x3D..0x3D,
-        0x40..0x5A,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def url_query
-      @url_query ||= from_ranges(
-        0x21..0x21,
-        0x24..0x24,
-        0x26..0x3B,
-        0x3D..0x3D,
-        0x3F..0x5A,
-        0x5F..0x5F,
-        0x61..0x7A,
-        0x7E..0x7E
-      ).freeze
-    end
-
-    def whitespace
-      @whitespace ||= from_ranges(
-        0x9..0x9,
-        0xA..0xD,
-        0x20..0x20,
-        0x85..0x85,
-        0xA0..0xA0,
-        0x1680..0x1680,
-        0x180E..0x180E,
-        0x2000..0x200A,
-        0x2028..0x2029,
-        0x202F..0x202F,
-        0x205F..0x205F,
-        0x3000..0x3000
-      ).freeze
-    end
-
-    def emoji
-      @emoji ||= from_ranges(
-        0x23..0x23,
-        0x2A..0x2A,
-        0x30..0x39,
-        0xA9..0xA9,
-        0xAE..0xAE,
-        0x203C..0x203C,
-        0x2049..0x2049,
-        0x2122..0x2122,
-        0x2139..0x2139,
-        0x2194..0x2199,
-        0x21A9..0x21AA,
-        0x231A..0x231B,
-        0x2328..0x2328,
-        0x23CF..0x23CF,
-        0x23E9..0x23F3,
-        0x23F8..0x23FA,
-        0x24C2..0x24C2,
-        0x25AA..0x25AB,
-        0x25B6..0x25B6,
-        0x25C0..0x25C0,
-        0x25FB..0x25FE,
-        0x2600..0x2604,
-        0x260E..0x260E,
-        0x2611..0x2611,
-        0x2614..0x2615,
-        0x2618..0x2618,
-        0x261D..0x261D,
-        0x2620..0x2620,
-        0x2622..0x2623,
-        0x2626..0x2626,
-        0x262A..0x262A,
-        0x262E..0x262F,
-        0x2638..0x263A,
-        0x2640..0x2640,
-        0x2642..0x2642,
-        0x2648..0x2653,
-        0x2660..0x2660,
-        0x2663..0x2663,
-        0x2665..0x2666,
-        0x2668..0x2668,
-        0x267B..0x267B,
-        0x267F..0x267F,
-        0x2692..0x2697,
-        0x2699..0x2699,
-        0x269B..0x269C,
-        0x26A0..0x26A1,
-        0x26AA..0x26AB,
-        0x26B0..0x26B1,
-        0x26BD..0x26BE,
-        0x26C4..0x26C5,
-        0x26C8..0x26C8,
-        0x26CE..0x26CF,
-        0x26D1..0x26D1,
-        0x26D3..0x26D4,
-        0x26E9..0x26EA,
-        0x26F0..0x26F5,
-        0x26F7..0x26FA,
-        0x26FD..0x26FD,
-        0x2702..0x2702,
-        0x2705..0x2705,
-        0x2708..0x270D,
-        0x270F..0x270F,
-        0x2712..0x2712,
-        0x2714..0x2714,
-        0x2716..0x2716,
-        0x271D..0x271D,
-        0x2721..0x2721,
-        0x2728..0x2728,
-        0x2733..0x2734,
-        0x2744..0x2744,
-        0x2747..0x2747,
-        0x274C..0x274C,
-        0x274E..0x274E,
-        0x2753..0x2755,
-        0x2757..0x2757,
-        0x2763..0x2764,
-        0x2795..0x2797,
-        0x27A1..0x27A1,
-        0x27B0..0x27B0,
-        0x27BF..0x27BF,
-        0x2934..0x2935,
-        0x2B05..0x2B07,
-        0x2B1B..0x2B1C,
-        0x2B50..0x2B50,
-        0x2B55..0x2B55,
-        0x3030..0x3030,
-        0x303D..0x303D,
-        0x3297..0x3297,
-        0x3299..0x3299,
-        0x1F004..0x1F004,
-        0x1F0CF..0x1F0CF,
-        0x1F170..0x1F171,
-        0x1F17E..0x1F17F,
-        0x1F18E..0x1F18E,
-        0x1F191..0x1F19A,
-        0x1F1E6..0x1F1FF,
-        0x1F201..0x1F202,
-        0x1F21A..0x1F21A,
-        0x1F22F..0x1F22F,
-        0x1F232..0x1F23A,
-        0x1F250..0x1F251,
-        0x1F300..0x1F321,
-        0x1F324..0x1F393,
-        0x1F396..0x1F397,
-        0x1F399..0x1F39B,
-        0x1F39E..0x1F3F0,
-        0x1F3F3..0x1F3F5,
-        0x1F3F7..0x1F4FD,
-        0x1F4FF..0x1F53D,
-        0x1F549..0x1F54E,
-        0x1F550..0x1F567,
-        0x1F56F..0x1F570,
-        0x1F573..0x1F57A,
-        0x1F587..0x1F587,
-        0x1F58A..0x1F58D,
-        0x1F590..0x1F590,
-        0x1F595..0x1F596,
-        0x1F5A4..0x1F5A5,
-        0x1F5A8..0x1F5A8,
-        0x1F5B1..0x1F5B2,
-        0x1F5BC..0x1F5BC,
-        0x1F5C2..0x1F5C4,
-        0x1F5D1..0x1F5D3,
-        0x1F5DC..0x1F5DE,
-        0x1F5E1..0x1F5E1,
-        0x1F5E3..0x1F5E3,
-        0x1F5E8..0x1F5E8,
-        0x1F5EF..0x1F5EF,
-        0x1F5F3..0x1F5F3,
-        0x1F5FA..0x1F64F,
-        0x1F680..0x1F6C5,
-        0x1F6CB..0x1F6D2,
-        0x1F6E0..0x1F6E5,
-        0x1F6E9..0x1F6E9,
-        0x1F6EB..0x1F6EC,
-        0x1F6F0..0x1F6F0,
-        0x1F6F3..0x1F6F8,
-        0x1F910..0x1F93A,
-        0x1F93C..0x1F93E,
-        0x1F940..0x1F945,
-        0x1F947..0x1F94C,
-        0x1F950..0x1F96B,
-        0x1F980..0x1F997,
-        0x1F9C0..0x1F9C0,
-        0x1F9D0..0x1F9E6
-      ).freeze
-    end
-
-    def respond_to_missing?(method_name, include_private = false)
-      (base = method_name[/^non_(.*)/, 1]) && respond_to?(base) || super
-    end
-
-    def method_missing(method_name, *args, &block)
-      if (base = method_name[/^non_(.*)/, 1])
-        ivar_name = "@#{method_name}"
-        return instance_variable_get(ivar_name) ||
-               instance_variable_set(ivar_name, send(base).inversion.freeze)
+    Dir[File.join(__dir__, 'predefined_sets', '*.cps')].each do |path|
+      set_name = File.basename(path, '.cps')
+
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
+        def #{set_name}
+          @#{set_name} ||= build_from_cps_file('#{path}').freeze
+        end
+
+        def non_#{set_name}
+          @non_#{set_name} ||= build_from_cps_file('#{path}').inversion.freeze
+        end
+      RUBY
+    end
+
+    alias all                      any
+    alias ascii_letters            ascii_letter
+    alias basic_multilingual_plane bmp
+    alias blank                    whitespace
+    alias invalid                  surrogate
+    alias valid                    unicode
+
+    def build_from_cps_file(path)
+      File.readlines(path).inject(new) do |set, line|
+        range_start, range_end = line.split(',')
+        set.merge((range_start.to_i(16))..(range_end.to_i(16)))
       end
-      super
     end
   end
 end

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -6,9 +6,13 @@ class CharacterSet
           new(Array(ranges).flat_map(&:to_a))
         end
 
-        def of(string)
-          raise ArgumentError, 'pass a String' unless string.is_a?(String)
-          new(string.codepoints)
+        def of(*strings)
+          new_set = new
+          strings.each do |str|
+            raise ArgumentError, 'pass a String' unless str.respond_to?(:codepoints)
+            str.codepoints.each { |cp| new_set << cp }
+          end
+          new_set
         end
       end
 
@@ -31,7 +35,7 @@ class CharacterSet
       end
 
       def ranges
-        CharacterSet.require_optional_dependency('range_compressor')
+        CharacterSet.require_optional_dependency('range_compressor', __method__)
         RangeCompressor.compress(self)
       end
 
@@ -39,9 +43,8 @@ class CharacterSet
         count.nil? ? to_a(true).sample : to_a(true).sample(count)
       end
 
-      def used_by?(string)
-        str!(string).each_codepoint { |cp| return true if include?(cp) }
-        false
+      def count_in(string)
+        str!(string).each_codepoint.count { |cp| include?(cp) }
       end
 
       def cover?(string)
@@ -67,16 +70,64 @@ class CharacterSet
         result.size == string.size ? nil : string.replace(result)
       end
 
+      def scan(string)
+        encoding = str!(string).encoding
+        string.each_codepoint.inject([]) do |arr, cp|
+          include?(cp) ? arr.push(cp.chr(encoding)) : arr
+        end
+      end
+
+      def used_by?(string)
+        str!(string).each_codepoint { |cp| return true if include?(cp) }
+        false
+      end
+
+      def section(from:, upto: 0x10FFFF)
+        dup.keep_if { |cp| cp >= from && cp <= upto }
+      end
+
+      def count_in_section(from:, upto: 0x10FFFF)
+        count { |cp| cp >= from && cp <= upto }
+      end
+
+      def section?(from:, upto: 0x10FFFF)
+        any? { |cp| cp >= from && cp <= upto }
+      end
+
+      def section_ratio(from:, upto: 0x10FFFF)
+        section(from: from, upto: upto).count / count.to_f
+      end
+
+      def planes
+        plane_size = 0x10000.to_f
+        inject({}) { |hash, cp| hash.merge((cp / plane_size).floor => 1) }.keys
+      end
+
+      def plane(num)
+        validate_plane_number(num)
+        section(from: (num * 0x10000), upto: ((num + 1) * 0x10000) - 1)
+      end
+
+      def member_in_plane?(num)
+        validate_plane_number(num)
+        ((num * 0x10000)...((num + 1) * 0x10000)).any? { |cp| include?(cp) }
+      end
+
       private
 
+      def validate_plane_number(num)
+        num >= 0 && num <= 16 or raise ArgumentError, 'plane must be between 0 and 16'
+      end
+
       def str!(obj)
         raise ArgumentError, 'pass a String' unless obj.respond_to?(:codepoints)
         obj
       end
 
       def make_new_str(original, &block)
-        new_string = str!(original).each_codepoint.each_with_object('', &block)
-        original.tainted? ? new_string.taint : new_string
+        str!(original)
+          .each_codepoint
+          .each_with_object(''.encode(original.encoding), &block)
       end
     end
   end

data/lib/character_set/ruby_fallback/set_methods.rb

@@ -1,7 +1,9 @@
 class CharacterSet
   module RubyFallback
     module SetMethods
-      Enumerable.instance_methods.concat(%w[empty? length size]).each do |mthd|
+      (Enumerable.instance_methods -
+        %i[include? member? to_a] +
+        %i[empty? length size]).each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(*args, &block)
             @__set.#{mthd}(*args, &block)
@@ -9,7 +11,7 @@ class CharacterSet
         RUBY
       end
 
-      %w[< <= > >= disjoint? intersect? proper_subset? proper_superset?
+      %i[< <= > >= disjoint? intersect? proper_subset? proper_superset?
          subset? superset?].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(enum, &block)
@@ -21,8 +23,8 @@ class CharacterSet
         RUBY
       end
 
-      %w[<< === add add? clear collect! delete delete? delete_if
-         each filter! hash include? map! member? keep_if reject!
+      %i[<< add add? clear collect! delete delete? delete_if
+         each filter! map! keep_if reject!
          select! subtract].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{mthd}(*args, &block)
@@ -32,22 +34,22 @@ class CharacterSet
         RUBY
       end
 
-      %w[& + - ^ | difference intersection union].each do |mthd|
+      # revert if https://github.com/knu/sorted_set/issues/2 is resolved
+      %i[=== include? member?].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{mthd}(enum, &block)
-            if enum.respond_to?(:map)
-              enum = enum.map { |el| el.is_a?(String) ? el.ord : el }
-            end
-            self.class.new(@__set.#{mthd}(enum, &block).to_a)
+          def #{mthd}(*args, &block)
+            !!@__set.#{mthd}(*args, &block)
           end
         RUBY
       end
 
-      %w[taint untaint].each do |mthd|
+      %i[& + - ^ | difference intersection union].each do |mthd|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{mthd}
-            @__set.#{mthd}
-            super
+          def #{mthd}(enum, &block)
+            if enum.respond_to?(:map)
+              enum = enum.map { |el| el.is_a?(String) ? el.ord : el }
+            end
+            self.class.new(@__set.#{mthd}(enum, &block).to_a)
           end
         RUBY
       end
@@ -72,8 +74,8 @@ class CharacterSet
           true
         elsif other.instance_of?(self.class)
           @__set == other.instance_variable_get(:@__set)
-        elsif other.is_a?(self.class) && size == other.size
-          other.all? { |cp| @__set.include?(cp) }
+        elsif other.is_a?(CharacterSet) || other.is_a?(CharacterSet::Pure)
+          size == other.size && other.all? { |cp| @__set.include?(cp) }
         else
           false
         end
@@ -81,7 +83,13 @@ class CharacterSet
 
       def eql?(other)
         return false unless other.is_a?(self.class)
-        @__set.eql?(other.instance_variable_get(:@__set))
+        # revert if https://github.com/knu/sorted_set/issues/3 is resolved
+        hash == other.hash
+      end
+
+      # revert if https://github.com/knu/sorted_set/issues/3 is resolved
+      def hash
+        @__set.to_a.hash
       end
 
       def initialize_dup(orig)