character_set 1.4.0 → 1.8.0

data/lib/character_set/ruby_fallback/vendored_set_classes.rb

@@ -0,0 +1,385 @@
+# set, vendored from https://github.com/ruby/set/blob/master/lib/set.rb,
+# with comments removed and linted.
+class CharacterSet::RubyFallback::Set
+  Set = self
+  include Enumerable
+
+  def self.[](*ary)
+    new(ary)
+  end
+
+  def initialize(enum = nil, &block)
+    @hash = Hash.new(false)
+
+    enum.nil? and return
+
+    if block
+      do_with_enum(enum) { |o| add(block[o]) }
+    else
+      merge(enum)
+    end
+  end
+
+  def do_with_enum(enum, &block)
+    if enum.respond_to?(:each_entry)
+      enum.each_entry(&block) if block
+    elsif enum.respond_to?(:each)
+      enum.each(&block) if block
+    else
+      raise ArgumentError, "value must be enumerable"
+    end
+  end
+  private :do_with_enum
+
+  def initialize_dup(orig)
+    super
+    @hash = orig.instance_variable_get(:@hash).dup
+  end
+
+  if Kernel.instance_method(:initialize_clone).arity != 1
+    def initialize_clone(orig, **options)
+      super
+      @hash = orig.instance_variable_get(:@hash).clone(**options)
+    end
+  else
+    def initialize_clone(orig)
+      super
+      @hash = orig.instance_variable_get(:@hash).clone
+    end
+  end
+
+  def freeze
+    @hash.freeze
+    super
+  end
+
+  def size
+    @hash.size
+  end
+  alias length size
+
+  def empty?
+    @hash.empty?
+  end
+
+  def clear
+    @hash.clear
+    self
+  end
+
+  def to_a
+    @hash.keys
+  end
+
+  def include?(o)
+    @hash[o]
+  end
+  alias member? include?
+
+  def superset?(set)
+    case
+    when set.instance_of?(self.class) && @hash.respond_to?(:>=)
+      @hash >= set.instance_variable_get(:@hash)
+    when set.is_a?(Set)
+      size >= set.size && set.all? { |o| include?(o) }
+    else
+      raise ArgumentError, "value must be a set"
+    end
+  end
+  alias >= superset?
+
+  def proper_superset?(set)
+    case
+    when set.instance_of?(self.class) && @hash.respond_to?(:>)
+      @hash > set.instance_variable_get(:@hash)
+    when set.is_a?(Set)
+      size > set.size && set.all? { |o| include?(o) }
+    else
+      raise ArgumentError, "value must be a set"
+    end
+  end
+  alias > proper_superset?
+
+  def subset?(set)
+    case
+    when set.instance_of?(self.class) && @hash.respond_to?(:<=)
+      @hash <= set.instance_variable_get(:@hash)
+    when set.is_a?(Set)
+      size <= set.size && all? { |o| set.include?(o) }
+    else
+      raise ArgumentError, "value must be a set"
+    end
+  end
+  alias <= subset?
+
+  def proper_subset?(set)
+    case
+    when set.instance_of?(self.class) && @hash.respond_to?(:<)
+      @hash < set.instance_variable_get(:@hash)
+    when set.is_a?(Set)
+      size < set.size && all? { |o| set.include?(o) }
+    else
+      raise ArgumentError, "value must be a set"
+    end
+  end
+  alias < proper_subset?
+
+  def <=>(set)
+    return unless set.is_a?(Set)
+
+    case size <=> set.size
+    when -1 then -1 if proper_subset?(set)
+    when +1 then +1 if proper_superset?(set)
+    else 0 if self.==(set)
+    end
+  end
+
+  def intersect?(set)
+    case set
+    when Set
+      if size < set.size
+        any? { |o| set.include?(o) }
+      else
+        set.any? { |o| include?(o) }
+      end
+    when Enumerable
+      set.any? { |o| include?(o) }
+    else
+      raise ArgumentError, "value must be enumerable"
+    end
+  end
+
+  def disjoint?(set)
+    !intersect?(set)
+  end
+
+  def each(&block)
+    block_given? or return enum_for(__method__) { size }
+    @hash.each_key(&block)
+    self
+  end
+
+  def add(o)
+    @hash[o] = true
+    self
+  end
+  alias << add
+
+  def add?(o)
+    add(o) unless include?(o)
+  end
+
+  def delete(o)
+    @hash.delete(o)
+    self
+  end
+
+  def delete?(o)
+    delete(o) if include?(o)
+  end
+
+  def delete_if
+    block_given? or return enum_for(__method__) { size }
+    select { |o| yield o }.each { |o| @hash.delete(o) }
+    self
+  end
+
+  def keep_if
+    block_given? or return enum_for(__method__) { size }
+    reject { |o| yield o }.each { |o| @hash.delete(o) }
+    self
+  end
+
+  def reject!(&block)
+    block_given? or return enum_for(__method__) { size }
+    n = size
+    delete_if(&block)
+    self if size != n
+  end
+
+  def select!(&block)
+    block_given? or return enum_for(__method__) { size }
+    n = size
+    keep_if(&block)
+    self if size != n
+  end
+
+  alias filter! select!
+
+  def merge(*enums, **_rest)
+    enums.each do |enum|
+      if enum.instance_of?(self.class)
+        @hash.update(enum.instance_variable_get(:@hash))
+      else
+        do_with_enum(enum) { |o| add(o) }
+      end
+    end
+
+    self
+  end
+
+  def subtract(enum)
+    do_with_enum(enum) { |o| delete(o) }
+    self
+  end
+
+  def |(enum)
+    dup.merge(enum)
+  end
+  alias + |
+  alias union |
+
+  def -(enum)
+    dup.subtract(enum)
+  end
+  alias difference -
+
+  def &(enum)
+    n = self.class.new
+    if enum.is_a?(Set)
+      if enum.size > size
+        each { |o| n.add(o) if enum.include?(o) }
+      else
+        enum.each { |o| n.add(o) if include?(o) }
+      end
+    else
+      do_with_enum(enum) { |o| n.add(o) if include?(o) }
+    end
+    n
+  end
+  alias intersection &
+
+  def ^(enum)
+    n = Set.new(enum)
+    each { |o| n.add(o) unless n.delete?(o) }
+    n
+  end
+
+  def ==(other)
+    if self.equal?(other)
+      true
+    elsif other.instance_of?(self.class)
+      @hash == other.instance_variable_get(:@hash)
+    elsif other.is_a?(Set) && self.size == other.size
+      other.all? { |o| @hash.include?(o) }
+    else
+      false
+    end
+  end
+
+  def hash
+    @hash.hash
+  end
+
+  def eql?(o)
+    return false unless o.is_a?(Set)
+    @hash.eql?(o.instance_variable_get(:@hash))
+  end
+
+  alias === include?
+
+  def classify
+    block_given? or return enum_for(__method__) { size }
+
+    h = {}
+
+    each { |i|
+      (h[yield(i)] ||= self.class.new).add(i)
+    }
+
+    h
+  end
+
+  def divide(&func)
+    func or return enum_for(__method__) { size }
+
+    if func.arity == 2
+      require 'tsort'
+
+      class << dig = {}
+        include TSort
+
+        alias tsort_each_node each_key
+        def tsort_each_child(node, &block)
+          fetch(node).each(&block)
+        end
+      end
+
+      each { |u|
+        dig[u] = a = []
+        each{ |v| func.call(u, v) and a << v }
+      }
+
+      set = Set.new()
+      dig.each_strongly_connected_component { |css|
+        set.add(self.class.new(css))
+      }
+      set
+    else
+      Set.new(classify(&func).values)
+    end
+  end
+end
+
+# sorted_set without rbtree dependency, vendored from
+# https://github.com/ruby/set/blob/72f08c4/lib/set.rb#L731-L800
+class CharacterSet::RubyFallback::SortedSet < CharacterSet::RubyFallback::Set
+  def initialize(*args)
+    @keys = nil
+    super
+  end
+
+  def clear
+    @keys = nil
+    super
+  end
+
+  def add(o)
+    @keys = nil
+    super
+  end
+  alias << add
+
+  def delete(o)
+    @keys = nil
+    @hash.delete(o)
+    self
+  end
+
+  def delete_if
+    block_given? or return enum_for(__method__) { size }
+    n = @hash.size
+    super
+    @keys = nil if @hash.size != n
+    self
+  end
+
+  def keep_if
+    block_given? or return enum_for(__method__) { size }
+    n = @hash.size
+    super
+    @keys = nil if @hash.size != n
+    self
+  end
+
+  def merge(enum)
+    @keys = nil
+    super
+  end
+
+  def each(&block)
+    block or return enum_for(__method__) { size }
+    to_a.each(&block)
+    self
+  end
+
+  def to_a
+    (@keys = @hash.keys).sort! unless @keys
+    @keys.dup
+  end
+
+  def freeze
+    to_a
+    super
+  end
+end

data/lib/character_set/ruby_fallback.rb

@@ -1,4 +1,3 @@
-require 'set'
 require 'character_set/ruby_fallback/set_methods'
 require 'character_set/ruby_fallback/character_set_methods'
 
@@ -12,8 +11,25 @@ class CharacterSet
     end
 
     def initialize(enum = [])
-      @__set = SortedSet.new
+      @__set = CharacterSet::RubyFallback::SortedSet.new
       super
     end
   end
 end
+
+if RUBY_PLATFORM[/java/i]
+  # JRuby has sorted_set in the stdlib.
+  require 'set'
+  CharacterSet::RubyFallback::Set       = ::Set
+  CharacterSet::RubyFallback::SortedSet = ::SortedSet
+else
+  # For other rubies, set/sorted_set are vendored due to dependency issues:
+  #
+  # - issues with default vs. installed gems such as [#2]
+  # - issues with the sorted_set dependency rb_tree
+  # - long-standing issues in recent versions of sorted_set
+  #
+  # The RubyFallback, and thus these set classes, are only used for testing,
+  # and for exotic rubies which use neither C nor Java.
+  require 'character_set/ruby_fallback/vendored_set_classes'
+end

data/lib/character_set/set_method_adapters.rb

@@ -22,13 +22,14 @@ class CharacterSet
 
     # Allow some methods to take an Enum just as well as another CharacterSet.
     # Tested by ruby-spec.
-    %w[& + - ^ | difference intersection subtract union].each do |method|
+    %w[& + - ^ | <=> difference disjoint? intersect? intersection
+       subtract union].each do |method|
       class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{method}(arg)
           if arg.is_a?(CharacterSet)
-            super
+            super(arg)
           elsif arg.respond_to?(:each)
-            super(CharacterSet.new(arg.to_a))
+            super(self.class.new(arg.to_a))
           else
             raise ArgumentError, 'pass an enumerable'
           end

data/lib/character_set/shared_methods.rb

@@ -15,6 +15,12 @@ class CharacterSet
             new(Array(args))
           end
 
+          def of(*args)
+            args.map do |arg|
+              arg.is_a?(Regexp) ? of_regexp(arg) : of_string(arg)
+            end.reduce(:merge) || new
+          end
+
           def parse(string)
             codepoints = Parser.codepoints_from_bracket_expression(string)
             result = new(codepoints)
@@ -22,33 +28,29 @@ class CharacterSet
           end
 
           def of_property(property_name)
-            require_optional_dependency('regexp_property_values')
+            require_optional_dependency('regexp_property_values', __method__)
 
             property = RegexpPropertyValues[property_name.to_s]
             from_ranges(*property.matched_ranges)
           end
 
           def of_regexp(regexp)
-            require_optional_dependency('regexp_parser')
+            require_optional_dependency('regexp_parser', __method__)
 
             root = ::Regexp::Parser.parse(regexp)
             of_expression(root)
           end
 
           def of_expression(expression)
-            ExpressionConverter.convert(expression)
+            ExpressionConverter.convert(expression, self)
           end
 
-          def require_optional_dependency(name)
+          def require_optional_dependency(name, method)
             required_optional_dependencies[name] ||= begin
               require name
               true
             rescue ::LoadError
-              entry_point = caller_locations.reverse.find do |loc|
-                loc.absolute_path.to_s.include?('/lib/character_set')
-              end
-              method = entry_point && entry_point.label
-              raise LoadError, 'You must the install the optional dependency '\
+              raise LoadError, 'You must install the optional dependency '\
                                "'\#{name}' to use the method `\#{method}'."
             end
           end
@@ -94,6 +96,14 @@ class CharacterSet
           Writer.write_surrogate_alternation(bmp_part.ranges, astral_part.ranges)
         end
 
+        def secure_token(length = 32)
+          CharacterSet.require_optional_dependency('securerandom', __method__)
+          cps = to_a
+          len = cps.count
+          1.upto(length).map { cps[SecureRandom.random_number(len)] }.pack('U*')
+        end
+        alias random_token secure_token
+
         def inspect
           len = length
           "#<#{klass.name}: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"
@@ -155,8 +165,12 @@ class CharacterSet
         end
 
         def divide(&func)
-          require 'set'
-          Set.new(to_a).divide(&func)
+          require 'character_set/ruby_fallback'
+          CharacterSet::RubyFallback::Set.new(to_a).divide(&func)
+        end
+
+        def join(separator = '')
+          to_a(true).join(separator)
         end
       RUBY
 

data/lib/character_set/version.rb

@@ -1,3 +1,3 @@
 class CharacterSet
-  VERSION = '1.4.0'
+  VERSION = '1.8.0'
 end

data/tasks/benchmark.rake

@@ -0,0 +1,20 @@
+desc 'Run all IPS benchmarks'
+task :benchmark do
+  Dir["#{__dir__}/benchmarks/*.rb"].sort.each { |file| load(file) }
+end
+
+namespace :benchmark do
+  desc 'Run all IPS benchmarks and store the comparison results in BENCHMARK.md'
+  task :write_to_file do
+    Rake.application[:benchmark].invoke
+
+    # extract comparison results from reports
+    results = $benchmark_results
+      .map { |caption, report| "```\n#{caption}\n\n#{report[/(?<=Comparison:).+/m].strip}\n```" }
+      .join("\n")
+      .gsub(/ \(±[^)]+\) |(?<=same-ish).*/, '') # remove some noise
+
+    File.write "#{__dir__}/../BENCHMARK.md",
+               "Results of `rake:benchmark` on #{RUBY_DESCRIPTION}\n\n#{results}\n"
+  end
+end

data/{benchmarks → tasks/benchmarks}/delete_in.rb

@@ -2,24 +2,28 @@ require_relative './shared'
 
 str = 'Lorem     ipsum       et      dolorem'
 rx = /\s/
+trt = "\t\n\v\f\r\s"
 cs = CharacterSet.whitespace
 
 benchmark(
-  caption: 'Removing whitespace',
+  caption: 'Removing ASCII whitespace',
   cases: {
     'String#gsub'            => -> { str.gsub(rx, '') },
+    'String#tr'              => -> { str.tr(trt, '') },
     'CharacterSet#delete_in' => -> { cs.delete_in(str) },
   }
 )
 
 str = 'Lörem ipsüm ⛷ et dölörem'
 rx = /[\s\p{emoji}äüö]/
+trt = "\t\n\v\f\r\s😀-🙏äüö"
 cs = CharacterSet.whitespace + CharacterSet.emoji + CharacterSet['ä', 'ö', 'ü']
 
 benchmark(
   caption: 'Removing whitespace, emoji and umlauts',
   cases: {
     'String#gsub'            => -> { str.gsub(rx, '') },
+    'String#tr'              => -> { str.tr(trt, '') },
     'CharacterSet#delete_in' => -> { cs.delete_in(str) },
   }
 )

data/{benchmarks → tasks/benchmarks}/keep_in.rb

@@ -2,24 +2,28 @@ require_relative './shared'
 
 str = 'Lorem ipsum et dolorem'
 rx = /\S/
+trt = "\u{0080}-\u{10FFFF}" # approximation
 cs = CharacterSet.whitespace
 
 benchmark(
   caption: 'Removing non-whitespace',
   cases: {
     'String#gsub'          => -> { str.gsub(rx, '') },
+    'String#tr'            => -> { str.tr(trt, '') },
     'CharacterSet#keep_in' => -> { cs.keep_in(str) },
   }
 )
 
 str = 'Lorem ipsum ⛷ et dolorem'
 rx = /\p{^emoji}/
+trt = "\u0000-\u{1F599}\u{1F650}-\u{10FFFF}"
 cs = CharacterSet.emoji
 
 benchmark(
-  caption: 'Extracting emoji',
+  caption: 'Keeping only emoji',
   cases: {
     'String#gsub'          => -> { str.gsub(rx, '') },
+    'String#tr'            => -> { str.tr(trt, '') },
     'CharacterSet#keep_in' => -> { cs.keep_in(str) },
   }
 )

data/tasks/benchmarks/shared.rb

@@ -0,0 +1,28 @@
+require 'benchmark/ips'
+require_relative '../../lib/character_set'
+if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+  require 'sorted_set'
+else
+  require 'set'
+end
+
+def benchmark(caption: nil, cases: {})
+  with_stdouts($stdout, string_io = StringIO.new) do
+    puts caption
+    Benchmark.ips do |x|
+      cases.each { |label, callable| x.report(label, &callable) }
+      x.compare!
+    end
+  end
+  ($benchmark_results ||= {})[caption] = string_io.string
+end
+
+def with_stdouts(*ios)
+  old_stdout = $stdout
+  ios.define_singleton_method(:method_missing) { |*args| each { |io| io.send(*args) } }
+  ios.define_singleton_method(:respond_to?) { |*args| IO.respond_to?(*args) }
+  $stdout = ios
+  yield
+ensure
+  $stdout = old_stdout
+end

data/tasks/sync_casefold_data.rake

@@ -0,0 +1,20 @@
+desc 'Download unicode casefold data and write new C header file'
+task :sync_casefold_data do
+  src_path = './CaseFolding.txt'
+  dst_path = "#{__dir__}/../ext/character_set/unicode_casefold_table.h"
+
+  `wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt`
+
+  mapping = File.foreach(src_path).each_with_object({}) do |line, hash|
+    from, type, to = line.split(/\s*;\s*/).first(3)
+    # type 'C' stands for 'common', excludes mappings to multiple chars
+    hash[from] = to if type == 'C'
+  end.sort
+
+  content = File.read(dst_path + '.tmpl')
+    .sub(/(CASEFOLD_COUNT )0/, "\\1#{mapping.count}")
+    .sub('{}', ['{', mapping.map { |a, b| "{0x#{a},0x#{b}}," }, '}'].join("\n"))
+
+  File.write(dst_path, content)
+  File.unlink(src_path)
+end

data/tasks/sync_predefined_sets.rake

@@ -0,0 +1,9 @@
+desc 'Update codepoint data for predefined sets, based on Onigmo'
+task :sync_predefined_sets do
+  %w[assigned emoji whitespace].each do |prop|
+    require 'regexp_property_values'
+    ranges = RegexpPropertyValues[prop].matched_ranges
+    str = ranges.map { |r| "#{r.min.to_s(16)},#{r.max.to_s(16)}\n" }.join.upcase
+    File.write("#{__dir__}/../lib/character_set/predefined_sets/#{prop}.cps", str, mode: 'w')
+  end
+end