character_set 1.2.0 → 1.3.0

data/lib/character_set/predefined_sets/bmp.cps

@@ -0,0 +1,2 @@
+0,D7FF
+E000,FFFF

data/lib/character_set/predefined_sets/crypt.cps

@@ -0,0 +1,2 @@
+2E,5A
+61,7A

data/lib/character_set/predefined_sets/emoji.cps

@@ -0,0 +1,151 @@
+23,23
+2A,2A
+30,39
+A9,A9
+AE,AE
+203C,203C
+2049,2049
+2122,2122
+2139,2139
+2194,2199
+21A9,21AA
+231A,231B
+2328,2328
+23CF,23CF
+23E9,23F3
+23F8,23FA
+24C2,24C2
+25AA,25AB
+25B6,25B6
+25C0,25C0
+25FB,25FE
+2600,2604
+260E,260E
+2611,2611
+2614,2615
+2618,2618
+261D,261D
+2620,2620
+2622,2623
+2626,2626
+262A,262A
+262E,262F
+2638,263A
+2640,2640
+2642,2642
+2648,2653
+265F,2660
+2663,2663
+2665,2666
+2668,2668
+267B,267B
+267E,267F
+2692,2697
+2699,2699
+269B,269C
+26A0,26A1
+26AA,26AB
+26B0,26B1
+26BD,26BE
+26C4,26C5
+26C8,26C8
+26CE,26CF
+26D1,26D1
+26D3,26D4
+26E9,26EA
+26F0,26F5
+26F7,26FA
+26FD,26FD
+2702,2702
+2705,2705
+2708,270D
+270F,270F
+2712,2712
+2714,2714
+2716,2716
+271D,271D
+2721,2721
+2728,2728
+2733,2734
+2744,2744
+2747,2747
+274C,274C
+274E,274E
+2753,2755
+2757,2757
+2763,2764
+2795,2797
+27A1,27A1
+27B0,27B0
+27BF,27BF
+2934,2935
+2B05,2B07
+2B1B,2B1C
+2B50,2B50
+2B55,2B55
+3030,3030
+303D,303D
+3297,3297
+3299,3299
+1F004,1F004
+1F0CF,1F0CF
+1F170,1F171
+1F17E,1F17F
+1F18E,1F18E
+1F191,1F19A
+1F1E6,1F1FF
+1F201,1F202
+1F21A,1F21A
+1F22F,1F22F
+1F232,1F23A
+1F250,1F251
+1F300,1F321
+1F324,1F393
+1F396,1F397
+1F399,1F39B
+1F39E,1F3F0
+1F3F3,1F3F5
+1F3F7,1F4FD
+1F4FF,1F53D
+1F549,1F54E
+1F550,1F567
+1F56F,1F570
+1F573,1F57A
+1F587,1F587
+1F58A,1F58D
+1F590,1F590
+1F595,1F596
+1F5A4,1F5A5
+1F5A8,1F5A8
+1F5B1,1F5B2
+1F5BC,1F5BC
+1F5C2,1F5C4
+1F5D1,1F5D3
+1F5DC,1F5DE
+1F5E1,1F5E1
+1F5E3,1F5E3
+1F5E8,1F5E8
+1F5EF,1F5EF
+1F5F3,1F5F3
+1F5FA,1F64F
+1F680,1F6C5
+1F6CB,1F6D2
+1F6D5,1F6D5
+1F6E0,1F6E5
+1F6E9,1F6E9
+1F6EB,1F6EC
+1F6F0,1F6F0
+1F6F3,1F6FA
+1F7E0,1F7EB
+1F90D,1F93A
+1F93C,1F945
+1F947,1F971
+1F973,1F976
+1F97A,1F9A2
+1F9A5,1F9AA
+1F9AE,1F9CA
+1F9CD,1F9FF
+1FA70,1FA73
+1FA78,1FA7A
+1FA80,1FA82
+1FA90,1FA95

data/lib/character_set/predefined_sets/newline.cps

@@ -0,0 +1,3 @@
+A,D
+85,85
+2028,2029

data/lib/character_set/predefined_sets/surrogate.cps

@@ -0,0 +1 @@
+D800,DFFF

data/lib/character_set/predefined_sets/unicode.cps

@@ -0,0 +1,2 @@
+0,D7FF
+E000,10FFFF

data/lib/character_set/predefined_sets/url_fragment.cps

@@ -0,0 +1,8 @@
+21,21
+24,24
+26,3B
+3D,3D
+3F,5A
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/url_host.cps

@@ -0,0 +1,10 @@
+21,21
+24,24
+26,2E
+30,3B
+3D,3D
+41,5B
+5D,5D
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/url_path.cps

@@ -0,0 +1,7 @@
+21,21
+24,3A
+3D,3D
+40,5A
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/url_query.cps

@@ -0,0 +1,8 @@
+21,21
+24,24
+26,3B
+3D,3D
+3F,5A
+5F,5F
+61,7A
+7E,7E

data/lib/character_set/predefined_sets/whitespace.cps

@@ -0,0 +1,10 @@
+9,D
+20,20
+85,85
+A0,A0
+1680,1680
+2000,200A
+2028,2029
+202F,202F
+205F,205F
+3000,3000

data/lib/character_set/ruby_fallback.rb

@@ -1,12 +1,10 @@
 require 'set'
 require 'character_set/ruby_fallback/set_methods'
-require 'character_set/ruby_fallback/plane_methods'
 require 'character_set/ruby_fallback/character_set_methods'
 
 class CharacterSet
   module RubyFallback
     include CharacterSet::RubyFallback::SetMethods
-    include CharacterSet::RubyFallback::PlaneMethods
     include CharacterSet::RubyFallback::CharacterSetMethods
 
     def self.prepended(klass)

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -39,9 +39,8 @@ class CharacterSet
         count.nil? ? to_a(true).sample : to_a(true).sample(count)
       end
 
-      def used_by?(string)
-        str!(string).each_codepoint { |cp| return true if include?(cp) }
-        false
+      def count_in(string)
+        str!(string).each_codepoint.count { |cp| include?(cp) }
       end
 
       def cover?(string)
@@ -67,15 +66,64 @@ class CharacterSet
         result.size == string.size ? nil : string.replace(result)
       end
 
+      def scan(string)
+        encoding = str!(string).encoding
+        string.each_codepoint.inject([]) do |arr, cp|
+          include?(cp) ? arr.push(cp.chr(encoding)) : arr
+        end
+      end
+
+      def used_by?(string)
+        str!(string).each_codepoint { |cp| return true if include?(cp) }
+        false
+      end
+
+      def section(from:, upto: 0x10FFFF)
+        dup.keep_if { |cp| cp >= from && cp <= upto }
+      end
+
+      def count_in_section(from:, upto: 0x10FFFF)
+        count { |cp| cp >= from && cp <= upto }
+      end
+
+      def section?(from:, upto: 0x10FFFF)
+        any? { |cp| cp >= from && cp <= upto }
+      end
+
+      def section_ratio(from:, upto: 0x10FFFF)
+        section(from: from, upto: upto).count / count.to_f
+      end
+
+      def planes
+        plane_size = 0x10000.to_f
+        inject({}) { |hash, cp| hash.merge((cp / plane_size).floor => 1) }.keys
+      end
+
+      def plane(num)
+        validate_plane_number(num)
+        section(from: (num * 0x10000), upto: ((num + 1) * 0x10000) - 1)
+      end
+
+      def member_in_plane?(num)
+        validate_plane_number(num)
+        ((num * 0x10000)...((num + 1) * 0x10000)).any? { |cp| include?(cp) }
+      end
+
       private
 
+      def validate_plane_number(num)
+        num >= 0 && num <= 16 or raise ArgumentError, 'plane must be between 0 and 16'
+      end
+
       def str!(obj)
         raise ArgumentError, 'pass a String' unless obj.respond_to?(:codepoints)
         obj
       end
 
       def make_new_str(original, &block)
-        new_string = str!(original).each_codepoint.each_with_object('', &block)
+        new_string = str!(original)
+          .each_codepoint
+          .each_with_object(''.encode(original.encoding), &block)
         original.tainted? ? new_string.taint : new_string
       end
     end

data/lib/character_set/ruby_fallback/set_methods.rb

@@ -72,8 +72,8 @@ class CharacterSet
           true
         elsif other.instance_of?(self.class)
           @__set == other.instance_variable_get(:@__set)
-        elsif other.is_a?(self.class) && size == other.size
-          other.all? { |cp| @__set.include?(cp) }
+        elsif other.is_a?(CharacterSet) || other.is_a?(CharacterSet::Pure)
+          size == other.size && other.all? { |cp| @__set.include?(cp) }
         else
           false
         end

data/lib/character_set/shared_methods.rb

@@ -70,7 +70,17 @@ class CharacterSet
           merge(enum)
         end
 
-        # stringification methods
+        # CharacterSet-specific conversion methods
+
+        def assigned_part
+          self & self.class.assigned
+        end
+
+        def valid_part
+          self - self.class.surrogate
+        end
+
+        # CharacterSet-specific stringification methods
 
         def to_s(opts = {}, &block)
           Writer.write(ranges, opts, &block)
@@ -82,25 +92,30 @@ class CharacterSet
 
         def inspect
           len = length
-          "#<CharacterSet: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"
+          "#<#{klass.name}: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"
         end
 
-        # unicode-plane-related methods
+        # C-extension adapter methods. Need overriding in pure fallback.
+        # Parsing kwargs in C is slower, verbose, and kinda deprecated.
+
+        def inversion(include_surrogates: false, upto: 0x10FFFF)
+          ext_inversion(include_surrogates, upto)
+        end
 
-        def bmp_part?
-          !bmp_part.empty?
+        def section(from:, upto: 0x10FFFF)
+          ext_section(from, upto)
         end
 
-        def astral_part?
-          !astral_part.empty?
+        def count_in_section(from:, upto: 0x10FFFF)
+          ext_count_in_section(from, upto)
         end
 
-        def bmp_ratio
-          bmp_part.count / count.to_f
+        def section?(from:, upto: 0x10FFFF)
+          ext_section?(from, upto)
         end
 
-        def astral_ratio
-          astral_part.count / count.to_f
+        def section_ratio(from:, upto: 0x10FFFF)
+          ext_section_ratio(from, upto)
         end
 
         #
@@ -136,42 +151,38 @@ class CharacterSet
         end
 
         def divide(&func)
-          block_given? or return enum_for(__method__) { size }
           require 'set'
+          Set.new(to_a).divide(&func)
+        end
+      RUBY
 
-          if func.arity == 2
-            require 'tsort'
-
-            class << dig = {}
-              include TSort
+      # CharacterSet-specific section methods
 
-              alias tsort_each_node each_key
-              def tsort_each_child(node, &block)
-                fetch(node).each(&block)
-              end
-            end
+      {
+        ascii:  0..0x7F,
+        bmp:    0..0xFFFF,
+        astral: 0x10000..0x10FFFF,
+      }.each do |section_name, range|
+        klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{section_name}_part
+            section(from: #{range.begin}, upto: #{range.end})
+          end
 
-            each do |u|
-              dig[u] = a = []
-              each{ |v| a << v if yield(u, v) }
-            end
+          def #{section_name}_part?
+            section?(from: #{range.begin}, upto: #{range.end})
+          end
 
-            set = Set.new
-            dig.each_strongly_connected_component do |css|
-              set.add(self.class.new(css))
-            end
-            set
-          else
-            Set.new(classify(&func).values)
+          def #{section_name}_only?
+            #{range.begin == 0 ?
+              "!section?(from: #{range.end}, upto: 0x10FFFF)" :
+              "!section?(from: 0, upto: #{range.begin})"}
           end
-        end
 
-        # C-extension adapter method. Needs overriding in pure fallback.
-        # Parsing kwargs in C is slower, verbose, and kinda deprecated.
-        def inversion(include_surrogates: false, upto: 0x10FFFF)
-          ext_inversion(include_surrogates, upto)
-        end
-      RUBY
+          def #{section_name}_ratio
+            section_ratio(from: #{range.begin}, upto: #{range.end})
+          end
+        RUBY
+      end
     end # self.included
   end # SharedMethods
 end