character_set 1.0.0

data/lib/character_set/character.rb

@@ -0,0 +1,76 @@
+class CharacterSet
+  class Character
+    ENCODING = 'utf-8'.freeze
+    SAFELY_PRINTABLE = (0x21..0x7E).to_a - ['-', '[', '\\', ']', '^'].map(&:ord)
+
+    attr_accessor :codepoint
+
+    def initialize(codepoint)
+      case codepoint
+      when Integer then self.codepoint = codepoint
+      when String  then self.codepoint = codepoint.ord
+      else              raise ArgumentError, 'pass an Integer or String'
+      end
+    end
+
+    def to_s
+      codepoint.chr(ENCODING)
+    end
+
+    def hex
+      codepoint.to_s(16).upcase
+    end
+
+    def escape(opts = {})
+      return to_s if SAFELY_PRINTABLE.include?(codepoint) && !opts[:escape_all]
+
+      return yield(self) if block_given?
+
+      # https://billposer.org/Software/ListOfRepresentations.html
+      case opts[:format].to_s.downcase.delete('-_ ')
+      when '', 'default', 'es6', 'esnext', 'rb', 'ruby'
+        default_escape(opts)
+      when 'java', 'javascript', 'js'
+        default_escape(opts, false)
+      when 'capitalizableu', 'c#', 'csharp', 'd', 'python'
+        capitalizable_u_escape
+      when 'u+', 'uplus'
+        u_plus_escape
+      when 'literal', 'raw'
+        to_s
+      else
+        raise ArgumentError, "unsupported format: #{opts[:format].inspect}"
+      end
+    end
+
+    def plane
+      codepoint / 0x10000
+    end
+
+    private
+
+    def default_escape(opts, support_wide_hex = true)
+      if hex.length <= 2
+        '\\x' + hex.rjust(2, '0')
+      elsif hex.length <= 4
+        '\\u' + hex.rjust(4, '0')
+      elsif support_wide_hex
+        '\\u{' + hex + '}'
+      else
+        raise "#{opts[:format]} does not support escaping astral value #{hex}"
+      end
+    end
+
+    def capitalizable_u_escape
+      if hex.length <= 4
+        '\\u' + hex.rjust(4, '0')
+      else
+        '\\U' + hex.rjust(8, '0')
+      end
+    end
+
+    def u_plus_escape
+      'U+' + hex.rjust(4, '0')
+    end
+  end
+end

data/lib/character_set/common_sets.rb

@@ -0,0 +1,258 @@
+class CharacterSet
+  module CommonSets
+    def ascii
+      @ascii ||= from_ranges(0..0x7F).freeze
+    end
+
+    # basic multilingual plane
+    def bmp
+      @bmp ||= from_ranges(0..0xD7FF, 0xE000..0xFFFF).freeze
+    end
+
+    # ./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
+    def crypt
+      @crypt ||= from_ranges(0x2E..0x5A, 0x61..0x7A).freeze
+    end
+
+    def newline
+      @newline ||= from_ranges(0xA..0xD, 0x85..0x85, 0x2028..0x2029).freeze
+    end
+
+    def unicode
+      @unicode ||= from_ranges(0..0xD7FF, 0xE000..0x10FFFF).freeze
+    end
+
+    def url_fragment
+      @url_fragment ||= from_ranges(
+        0x21..0x21,
+        0x24..0x24,
+        0x26..0x3B,
+        0x3D..0x3D,
+        0x3F..0x5A,
+        0x5F..0x5F,
+        0x61..0x7A,
+        0x7E..0x7E
+      ).freeze
+    end
+
+    def url_host
+      @url_host ||= from_ranges(
+        0x21..0x21,
+        0x24..0x24,
+        0x26..0x2E,
+        0x30..0x3B,
+        0x3D..0x3D,
+        0x41..0x5B,
+        0x5D..0x5D,
+        0x5F..0x5F,
+        0x61..0x7A,
+        0x7E..0x7E
+      ).freeze
+    end
+
+    def url_path
+      @url_path ||= from_ranges(
+        0x21..0x21,
+        0x24..0x3A,
+        0x3D..0x3D,
+        0x40..0x5A,
+        0x5F..0x5F,
+        0x61..0x7A,
+        0x7E..0x7E
+      ).freeze
+    end
+
+    def url_query
+      @url_query ||= from_ranges(
+        0x21..0x21,
+        0x24..0x24,
+        0x26..0x3B,
+        0x3D..0x3D,
+        0x3F..0x5A,
+        0x5F..0x5F,
+        0x61..0x7A,
+        0x7E..0x7E
+      ).freeze
+    end
+
+    def whitespace
+      @whitespace ||= from_ranges(
+        0x9..0x9,
+        0xA..0xD,
+        0x20..0x20,
+        0x85..0x85,
+        0xA0..0xA0,
+        0x1680..0x1680,
+        0x180E..0x180E,
+        0x2000..0x200A,
+        0x2028..0x2029,
+        0x202F..0x202F,
+        0x205F..0x205F,
+        0x3000..0x3000
+      ).freeze
+    end
+
+    def emoji
+      @emoji ||= from_ranges(
+        0x23..0x23,
+        0x2A..0x2A,
+        0x30..0x39,
+        0xA9..0xA9,
+        0xAE..0xAE,
+        0x203C..0x203C,
+        0x2049..0x2049,
+        0x2122..0x2122,
+        0x2139..0x2139,
+        0x2194..0x2199,
+        0x21A9..0x21AA,
+        0x231A..0x231B,
+        0x2328..0x2328,
+        0x23CF..0x23CF,
+        0x23E9..0x23F3,
+        0x23F8..0x23FA,
+        0x24C2..0x24C2,
+        0x25AA..0x25AB,
+        0x25B6..0x25B6,
+        0x25C0..0x25C0,
+        0x25FB..0x25FE,
+        0x2600..0x2604,
+        0x260E..0x260E,
+        0x2611..0x2611,
+        0x2614..0x2615,
+        0x2618..0x2618,
+        0x261D..0x261D,
+        0x2620..0x2620,
+        0x2622..0x2623,
+        0x2626..0x2626,
+        0x262A..0x262A,
+        0x262E..0x262F,
+        0x2638..0x263A,
+        0x2640..0x2640,
+        0x2642..0x2642,
+        0x2648..0x2653,
+        0x2660..0x2660,
+        0x2663..0x2663,
+        0x2665..0x2666,
+        0x2668..0x2668,
+        0x267B..0x267B,
+        0x267F..0x267F,
+        0x2692..0x2697,
+        0x2699..0x2699,
+        0x269B..0x269C,
+        0x26A0..0x26A1,
+        0x26AA..0x26AB,
+        0x26B0..0x26B1,
+        0x26BD..0x26BE,
+        0x26C4..0x26C5,
+        0x26C8..0x26C8,
+        0x26CE..0x26CF,
+        0x26D1..0x26D1,
+        0x26D3..0x26D4,
+        0x26E9..0x26EA,
+        0x26F0..0x26F5,
+        0x26F7..0x26FA,
+        0x26FD..0x26FD,
+        0x2702..0x2702,
+        0x2705..0x2705,
+        0x2708..0x270D,
+        0x270F..0x270F,
+        0x2712..0x2712,
+        0x2714..0x2714,
+        0x2716..0x2716,
+        0x271D..0x271D,
+        0x2721..0x2721,
+        0x2728..0x2728,
+        0x2733..0x2734,
+        0x2744..0x2744,
+        0x2747..0x2747,
+        0x274C..0x274C,
+        0x274E..0x274E,
+        0x2753..0x2755,
+        0x2757..0x2757,
+        0x2763..0x2764,
+        0x2795..0x2797,
+        0x27A1..0x27A1,
+        0x27B0..0x27B0,
+        0x27BF..0x27BF,
+        0x2934..0x2935,
+        0x2B05..0x2B07,
+        0x2B1B..0x2B1C,
+        0x2B50..0x2B50,
+        0x2B55..0x2B55,
+        0x3030..0x3030,
+        0x303D..0x303D,
+        0x3297..0x3297,
+        0x3299..0x3299,
+        0x1F004..0x1F004,
+        0x1F0CF..0x1F0CF,
+        0x1F170..0x1F171,
+        0x1F17E..0x1F17F,
+        0x1F18E..0x1F18E,
+        0x1F191..0x1F19A,
+        0x1F1E6..0x1F1FF,
+        0x1F201..0x1F202,
+        0x1F21A..0x1F21A,
+        0x1F22F..0x1F22F,
+        0x1F232..0x1F23A,
+        0x1F250..0x1F251,
+        0x1F300..0x1F321,
+        0x1F324..0x1F393,
+        0x1F396..0x1F397,
+        0x1F399..0x1F39B,
+        0x1F39E..0x1F3F0,
+        0x1F3F3..0x1F3F5,
+        0x1F3F7..0x1F4FD,
+        0x1F4FF..0x1F53D,
+        0x1F549..0x1F54E,
+        0x1F550..0x1F567,
+        0x1F56F..0x1F570,
+        0x1F573..0x1F57A,
+        0x1F587..0x1F587,
+        0x1F58A..0x1F58D,
+        0x1F590..0x1F590,
+        0x1F595..0x1F596,
+        0x1F5A4..0x1F5A5,
+        0x1F5A8..0x1F5A8,
+        0x1F5B1..0x1F5B2,
+        0x1F5BC..0x1F5BC,
+        0x1F5C2..0x1F5C4,
+        0x1F5D1..0x1F5D3,
+        0x1F5DC..0x1F5DE,
+        0x1F5E1..0x1F5E1,
+        0x1F5E3..0x1F5E3,
+        0x1F5E8..0x1F5E8,
+        0x1F5EF..0x1F5EF,
+        0x1F5F3..0x1F5F3,
+        0x1F5FA..0x1F64F,
+        0x1F680..0x1F6C5,
+        0x1F6CB..0x1F6D2,
+        0x1F6E0..0x1F6E5,
+        0x1F6E9..0x1F6E9,
+        0x1F6EB..0x1F6EC,
+        0x1F6F0..0x1F6F0,
+        0x1F6F3..0x1F6F8,
+        0x1F910..0x1F93A,
+        0x1F93C..0x1F93E,
+        0x1F940..0x1F945,
+        0x1F947..0x1F94C,
+        0x1F950..0x1F96B,
+        0x1F980..0x1F997,
+        0x1F9C0..0x1F9C0,
+        0x1F9D0..0x1F9E6
+      ).freeze
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      (base = method_name[/^non_(.*)/, 1]) && respond_to?(base) || super
+    end
+
+    def method_missing(method_name, *args, &block)
+      if (base = method_name[/^non_(.*)/, 1])
+        ivar_name = "@#{method_name}"
+        return instance_variable_get(ivar_name) ||
+               instance_variable_set(ivar_name, send(base).inversion.freeze)
+      end
+      super
+    end
+  end
+end

data/lib/character_set/core_ext/regexp_ext.rb

@@ -0,0 +1,11 @@
+class CharacterSet
+  module CoreExt
+    module RegexpExt
+      def character_set
+        CharacterSet.of_regexp(self)
+      end
+    end
+  end
+end
+
+::Regexp.send(:include, CharacterSet::CoreExt::RegexpExt)

data/lib/character_set/core_ext/string_ext.rb

@@ -0,0 +1,35 @@
+class CharacterSet
+  module CoreExt
+    module StringExt
+      def character_set
+        CharacterSet.of(self)
+      end
+
+      def covered_by_character_set?(set)
+        set.cover?(self)
+      end
+
+      def uses_character_set?(set)
+        set.used_by?(self)
+      end
+
+      def delete_character_set(set)
+        set.delete_in(self)
+      end
+
+      def delete_character_set!(set)
+        set.delete_in!(self)
+      end
+
+      def keep_character_set(set)
+        set.keep_in(self)
+      end
+
+      def keep_character_set!(set)
+        set.keep_in!(self)
+      end
+    end
+  end
+end
+
+::String.send(:include, CharacterSet::CoreExt::StringExt)

data/lib/character_set/core_ext.rb

@@ -0,0 +1,3 @@
+require 'character_set'
+require 'character_set/core_ext/regexp_ext'
+require 'character_set/core_ext/string_ext'

data/lib/character_set/expression_converter.rb

@@ -0,0 +1,106 @@
+class CharacterSet
+  module ExpressionConverter
+    module_function
+
+    Error = Class.new(ArgumentError)
+
+    def convert(expression)
+      @regexp_parser_required ||= require 'regexp_parser'
+
+      case expression
+      when Regexp::Expression::Root
+        if expression.count != 1
+          raise Error, 'Pass a Regexp with exactly one expression, e.g. /[a-z]/'
+        end
+        convert(expression[0])
+
+      when Regexp::Expression::CharacterSet
+        content = expression.map { |subexp| convert(subexp) }.reduce(:+)
+        expression.negative? ? content.inversion : content
+
+      when Regexp::Expression::CharacterSet::Intersection
+        expression.map { |subexp| convert(subexp) }.reduce(:&)
+
+      when Regexp::Expression::CharacterSet::IntersectedSequence
+        expression.map { |subexp| convert(subexp) }.reduce(:+)
+
+      when Regexp::Expression::CharacterSet::Range
+        start, finish = expression.map { |subexp| convert(subexp) }
+        CharacterSet.from_ranges((start.min)..(finish.max))
+
+      when Regexp::Expression::CharacterType::Any
+        CharacterSet.unicode
+
+      when Regexp::Expression::CharacterType::Digit
+        CharacterSet.from_ranges(48..57)
+
+      when Regexp::Expression::CharacterType::NonDigit
+        CharacterSet.from_ranges(48..57).inversion
+
+      when Regexp::Expression::CharacterType::Hex
+        CharacterSet.from_ranges(48..57, 65..70, 97..102)
+
+      when Regexp::Expression::CharacterType::NonHex
+        CharacterSet.from_ranges(48..57, 65..70, 97..102).inversion
+
+      when Regexp::Expression::CharacterType::Space
+        CharacterSet["\t", "\n", "\v", "\f", "\r", "\x20"]
+
+      when Regexp::Expression::CharacterType::NonSpace
+        CharacterSet["\t", "\n", "\v", "\f", "\r", "\x20"].inversion
+
+      when Regexp::Expression::CharacterType::Word
+        CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122)
+
+      when Regexp::Expression::CharacterType::NonWord
+        CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122).inversion
+
+      when Regexp::Expression::EscapeSequence::CodepointList
+        CharacterSet.new(expression.codepoints)
+
+      when Regexp::Expression::EscapeSequence::Base
+        CharacterSet[expression.codepoint]
+
+      when Regexp::Expression::Group::Capture,
+           Regexp::Expression::Group::Passive,
+           Regexp::Expression::Group::Named,
+           Regexp::Expression::Group::Atomic,
+           Regexp::Expression::Group::Options
+        case expression.count
+        when 0 then CharacterSet[]
+        when 1 then convert(expression.first)
+        else
+          raise Error, 'Groups must contain exactly one expression, e.g. ([a-z])'
+        end
+
+      when Regexp::Expression::Alternation
+        expression.map { |subexp| convert(subexp) }.reduce(:+)
+
+      when Regexp::Expression::Alternative
+        case expression.count
+        when 0 then CharacterSet[]
+        when 1 then convert(expression.first)
+        else
+          raise Error, 'Alternatives must contain exactly one expression'
+        end
+
+      when Regexp::Expression::Literal
+        if expression.set_level == 0 && expression.text.size != 1
+          raise Error, 'Literal runs outside of sets are codepoint *sequences*'
+        end
+        CharacterSet[expression.text.ord]
+
+      when Regexp::Expression::UnicodeProperty::Base,
+           Regexp::Expression::PosixClass
+        content = CharacterSet.of_property(expression.token)
+        expression.negative? ? content.inversion : content
+
+      when Regexp::Expression::Base
+        raise Error, "Unsupported expression class `#{expression.class}`"
+
+      else
+        raise Error, "Pass an expression (result of Regexp::Parser.parse)"
+      end
+    end
+  end
+end

data/lib/character_set/parser.rb

@@ -0,0 +1,48 @@
+class CharacterSet
+  module Parser
+    module_function
+
+    def codepoints_from_enumerable(object)
+      raise ArgumentError, 'pass an Enumerable' unless object.respond_to?(:each)
+      # Use #each to check first element (only this works for all Enumerables)
+      object.each do |e|
+        return object            if e.is_a?(Integer) && e >= 0 && e < 0x110000
+        return object.map(&:ord) if e.is_a?(String)  && e.length == 1
+        raise ArgumentError, "#{e.inspect} is not valid as a codepoint"
+      end
+    end
+
+    def codepoints_from_bracket_expression(string)
+      raise ArgumentError, 'pass a String'   unless string.is_a?(String)
+      raise ArgumentError, 'advanced syntax' if string =~ /\\[^uUx]|[^\\]\[|&&/
+
+      content = strip_brackets(string)
+      literal_content = eval_escapes(content)
+
+      prev_chr = nil
+      in_range = false
+
+      literal_content.each_char.map do |chr|
+        if chr == '-' && prev_chr && prev_chr != '\\' && prev_chr != '-'
+          in_range = true
+          nil
+        else
+          result = in_range ? ((prev_chr.ord + 1)..(chr.ord)).to_a : chr.ord
+          in_range = false
+          prev_chr = chr
+          result
+        end
+      end.compact.flatten
+    end
+
+    def strip_brackets(string)
+      string[/\A\[\^?(.*)\]\z/, 1] || string.dup
+    end
+
+    def eval_escapes(string)
+      string.gsub(/\\U(\h{8})|\\u(\h{4})|U\+(\h+)|\\x(\h{2})|\\u\{(\h+)\}/) do
+        ($1 || $2 || $3 || $4 || $5).to_i(16).chr('utf-8')
+      end
+    end
+  end
+end

data/lib/character_set/pure.rb

@@ -0,0 +1,13 @@
+require 'character_set'
+require 'character_set/ruby_fallback'
+
+# CharacterSet::Pure uses only Ruby implementations.
+# It is equal to CharacterSet if the C ext can't be loaded.
+class CharacterSet
+  class Pure
+    prepend CharacterSet::RubyFallback
+    prepend CharacterSet::SetMethodAdapters
+    include CharacterSet::SharedMethods
+    extend CharacterSet::CommonSets
+  end
+end

data/lib/character_set/ruby_fallback/character_set_methods.rb

@@ -0,0 +1,83 @@
+class CharacterSet
+  module RubyFallback
+    module CharacterSetMethods
+      module ClassMethods
+        def from_ranges(*ranges)
+          new(Array(ranges).flat_map(&:to_a))
+        end
+
+        def of(string)
+          raise ArgumentError, 'pass a String' unless string.is_a?(String)
+          new(string.codepoints)
+        end
+      end
+
+      def inversion(include_surrogates: false, upto: 0x10FFFF)
+        new_set = self.class.new
+        0.upto(upto) do |cp|
+          next unless include_surrogates || cp > 0xDFFF || cp < 0xD800
+          new_set << cp unless include?(cp)
+        end
+        new_set
+      end
+
+      def case_insensitive
+        new_set = dup
+        each do |cp|
+          swapped_cps = cp.chr('utf-8').swapcase.codepoints
+          swapped_cps.size == 1 && new_set << swapped_cps[0]
+        end
+        new_set
+      end
+
+      def ranges
+        @range_compressor_required ||= require 'range_compressor'
+        RangeCompressor.compress(self)
+      end
+
+      def sample(count = nil)
+        count.nil? ? to_a(true).sample : to_a(true).sample(count)
+      end
+
+      def used_by?(string)
+        str!(string).each_codepoint { |cp| return true if include?(cp) }
+        false
+      end
+
+      def cover?(string)
+        str!(string).each_codepoint { |cp| return false unless include?(cp) }
+        true
+      end
+
+      def delete_in(string)
+        make_new_str(string) { |cp, new_str| include?(cp) || (new_str << cp) }
+      end
+
+      def delete_in!(string)
+        result = delete_in(string)
+        result.size == string.size ? nil : string.replace(result)
+      end
+
+      def keep_in(string)
+        make_new_str(string) { |cp, new_str| include?(cp) && (new_str << cp) }
+      end
+
+      def keep_in!(string)
+        result = keep_in(string)
+        result.size == string.size ? nil : string.replace(result)
+      end
+
+      private
+
+      def str!(obj)
+        raise ArgumentError, 'pass a String' unless obj.respond_to?(:codepoints)
+        obj
+      end
+
+      def make_new_str(original, &block)
+        new_string = str!(original).each_codepoint.each_with_object('', &block)
+        original.tainted? ? new_string.taint : new_string
+      end
+    end
+  end
+end

data/lib/character_set/ruby_fallback/plane_methods.rb

@@ -0,0 +1,27 @@
+class CharacterSet
+  module RubyFallback
+    module PlaneMethods
+      def bmp_part
+        dup.keep_if { |cp| cp < 0x10000 }
+      end
+
+      def astral_part
+        dup.keep_if { |cp| cp >= 0x10000 }
+      end
+
+      def planes
+        plane_set = {}
+        plane_size = 0x10000.to_f
+        each do |cp|
+          plane = (cp / plane_size).floor
+          plane_set[plane] = true
+        end
+        plane_set.keys
+      end
+
+      def member_in_plane?(num)
+        ((num * 0x10000)...((num + 1) * 0x10000)).any? { |cp| include?(cp) }
+      end
+    end
+  end
+end