character_set 1.6.0 → 1.7.0

data/lib/character_set/ruby_fallback/vendored_set_classes.rb

@@ -0,0 +1,492 @@
+# set and sorted_set are vendored due to various dependency issues:
+#
+# - issues with default vs. installed gems such as [#2]
+# - issues with the sorted_set dependency rb_tree
+# - long-standing issues in recent versions of sorted_set
+#
+# The RubyFallback (and thus these set classes), are only used for testing,
+# and for exotic rubies which use neither C nor Java.
+
+class CharacterSet
+  module RubyFallback
+    if RUBY_PLATFORM[/java/i]
+      # Vendoring is not needed for JRuby which has sorted_set in the stdlib.
+      require 'set'
+
+      Set       = ::Set
+      SortedSet = ::SortedSet
+    else
+      # set, vendored from https://github.com/ruby/set/blob/master/lib/set.rb,
+      # with comments removed and linted.
+      class Set
+        include Enumerable
+
+        def self.[](*ary)
+          new(ary)
+        end
+
+        def initialize(enum = nil, &block)
+          @hash = Hash.new(false)
+
+          enum.nil? and return
+
+          if block
+            do_with_enum(enum) { |o| add(block[o]) }
+          else
+            merge(enum)
+          end
+        end
+
+        def compare_by_identity
+          if @hash.respond_to?(:compare_by_identity)
+            @hash.compare_by_identity
+            self
+          else
+            raise NotImplementedError, "#{self.class.name}\##{__method__} is not implemented"
+          end
+        end
+
+        def compare_by_identity?
+          @hash.respond_to?(:compare_by_identity?) && @hash.compare_by_identity?
+        end
+
+        def do_with_enum(enum, &block)
+          if enum.respond_to?(:each_entry)
+            enum.each_entry(&block) if block
+          elsif enum.respond_to?(:each)
+            enum.each(&block) if block
+          else
+            raise ArgumentError, "value must be enumerable"
+          end
+        end
+        private :do_with_enum
+
+        def initialize_dup(orig)
+          super
+          @hash = orig.instance_variable_get(:@hash).dup
+        end
+
+        if Kernel.instance_method(:initialize_clone).arity != 1
+          def initialize_clone(orig, **options)
+            super
+            @hash = orig.instance_variable_get(:@hash).clone(**options)
+          end
+        else
+          def initialize_clone(orig)
+            super
+            @hash = orig.instance_variable_get(:@hash).clone
+          end
+        end
+
+        def freeze
+          @hash.freeze
+          super
+        end
+
+        def size
+          @hash.size
+        end
+        alias length size
+
+        def empty?
+          @hash.empty?
+        end
+
+        def clear
+          @hash.clear
+          self
+        end
+
+        def replace(enum)
+          if enum.instance_of?(self.class)
+            @hash.replace(enum.instance_variable_get(:@hash))
+            self
+          else
+            do_with_enum(enum)
+            clear
+            merge(enum)
+          end
+        end
+
+        def to_a
+          @hash.keys
+        end
+
+        def to_set(klass = Set, *args, &block)
+          return self if instance_of?(Set) && klass == Set && block.nil? && args.empty?
+          klass.new(self, *args, &block)
+        end
+
+        def flatten_merge(set, seen = Set.new)
+          set.each { |e|
+            if e.is_a?(Set)
+              if seen.include?(e_id = e.object_id)
+                raise ArgumentError, "tried to flatten recursive Set"
+              end
+
+              seen.add(e_id)
+              flatten_merge(e, seen)
+              seen.delete(e_id)
+            else
+              add(e)
+            end
+          }
+
+          self
+        end
+        protected :flatten_merge
+
+        def flatten
+          self.class.new.flatten_merge(self)
+        end
+
+        def flatten!
+          replace(flatten()) if any? { |e| e.is_a?(Set) }
+        end
+
+        def include?(o)
+          @hash[o]
+        end
+        alias member? include?
+
+        def superset?(set)
+          case
+          when set.instance_of?(self.class) && @hash.respond_to?(:>=)
+            @hash >= set.instance_variable_get(:@hash)
+          when set.is_a?(Set)
+            size >= set.size && set.all? { |o| include?(o) }
+          else
+            raise ArgumentError, "value must be a set"
+          end
+        end
+        alias >= superset?
+
+        def proper_superset?(set)
+          case
+          when set.instance_of?(self.class) && @hash.respond_to?(:>)
+            @hash > set.instance_variable_get(:@hash)
+          when set.is_a?(Set)
+            size > set.size && set.all? { |o| include?(o) }
+          else
+            raise ArgumentError, "value must be a set"
+          end
+        end
+        alias > proper_superset?
+
+        def subset?(set)
+          case
+          when set.instance_of?(self.class) && @hash.respond_to?(:<=)
+            @hash <= set.instance_variable_get(:@hash)
+          when set.is_a?(Set)
+            size <= set.size && all? { |o| set.include?(o) }
+          else
+            raise ArgumentError, "value must be a set"
+          end
+        end
+        alias <= subset?
+
+        def proper_subset?(set)
+          case
+          when set.instance_of?(self.class) && @hash.respond_to?(:<)
+            @hash < set.instance_variable_get(:@hash)
+          when set.is_a?(Set)
+            size < set.size && all? { |o| set.include?(o) }
+          else
+            raise ArgumentError, "value must be a set"
+          end
+        end
+        alias < proper_subset?
+
+        def <=>(set)
+          return unless set.is_a?(Set)
+
+          case size <=> set.size
+          when -1 then -1 if proper_subset?(set)
+          when +1 then +1 if proper_superset?(set)
+          else 0 if self.==(set)
+          end
+        end
+
+        def intersect?(set)
+          case set
+          when Set
+            if size < set.size
+              any? { |o| set.include?(o) }
+            else
+              set.any? { |o| include?(o) }
+            end
+          when Enumerable
+            set.any? { |o| include?(o) }
+          else
+            raise ArgumentError, "value must be enumerable"
+          end
+        end
+
+        def disjoint?(set)
+          !intersect?(set)
+        end
+
+        def each(&block)
+          block_given? or return enum_for(__method__) { size }
+          @hash.each_key(&block)
+          self
+        end
+
+        def add(o)
+          @hash[o] = true
+          self
+        end
+        alias << add
+
+        def add?(o)
+          add(o) unless include?(o)
+        end
+
+        def delete(o)
+          @hash.delete(o)
+          self
+        end
+
+        def delete?(o)
+          delete(o) if include?(o)
+        end
+
+        def delete_if
+          block_given? or return enum_for(__method__) { size }
+          select { |o| yield o }.each { |o| @hash.delete(o) }
+          self
+        end
+
+        def keep_if
+          block_given? or return enum_for(__method__) { size }
+          reject { |o| yield o }.each { |o| @hash.delete(o) }
+          self
+        end
+
+        def collect!
+          block_given? or return enum_for(__method__) { size }
+          set = self.class.new
+          each { |o| set << yield(o) }
+          replace(set)
+        end
+        alias map! collect!
+
+        def reject!(&block)
+          block_given? or return enum_for(__method__) { size }
+          n = size
+          delete_if(&block)
+          self if size != n
+        end
+
+        def select!(&block)
+          block_given? or return enum_for(__method__) { size }
+          n = size
+          keep_if(&block)
+          self if size != n
+        end
+
+        alias filter! select!
+
+        def merge(*enums, **_rest)
+          enums.each do |enum|
+            if enum.instance_of?(self.class)
+              @hash.update(enum.instance_variable_get(:@hash))
+            else
+              do_with_enum(enum) { |o| add(o) }
+            end
+          end
+
+          self
+        end
+
+        def subtract(enum)
+          do_with_enum(enum) { |o| delete(o) }
+          self
+        end
+
+        def |(enum)
+          dup.merge(enum)
+        end
+        alias + |
+        alias union |
+
+        def -(enum)
+          dup.subtract(enum)
+        end
+        alias difference -
+
+        def &(enum)
+          n = self.class.new
+          if enum.is_a?(Set)
+            if enum.size > size
+              each { |o| n.add(o) if enum.include?(o) }
+            else
+              enum.each { |o| n.add(o) if include?(o) }
+            end
+          else
+            do_with_enum(enum) { |o| n.add(o) if include?(o) }
+          end
+          n
+        end
+        alias intersection &
+
+        def ^(enum)
+          n = Set.new(enum)
+          each { |o| n.add(o) unless n.delete?(o) }
+          n
+        end
+
+        def ==(other)
+          if self.equal?(other)
+            true
+          elsif other.instance_of?(self.class)
+            @hash == other.instance_variable_get(:@hash)
+          elsif other.is_a?(Set) && self.size == other.size
+            other.all? { |o| @hash.include?(o) }
+          else
+            false
+          end
+        end
+
+        def hash
+          @hash.hash
+        end
+
+        def eql?(o)
+          return false unless o.is_a?(Set)
+          @hash.eql?(o.instance_variable_get(:@hash))
+        end
+
+        def reset
+          if @hash.respond_to?(:rehash)
+            @hash.rehash
+          else
+            raise FrozenError, "can't modify frozen #{self.class.name}" if frozen?
+          end
+          self
+        end
+        alias === include?
+
+        def classify
+          block_given? or return enum_for(__method__) { size }
+
+          h = {}
+
+          each { |i|
+            (h[yield(i)] ||= self.class.new).add(i)
+          }
+
+          h
+        end
+
+        def divide(&func)
+          func or return enum_for(__method__) { size }
+
+          if func.arity == 2
+            require 'tsort'
+
+            class << dig = {}
+              include TSort
+
+              alias tsort_each_node each_key
+              def tsort_each_child(node, &block)
+                fetch(node).each(&block)
+              end
+            end
+
+            each { |u|
+              dig[u] = a = []
+              each{ |v| func.call(u, v) and a << v }
+            }
+
+            set = Set.new()
+            dig.each_strongly_connected_component { |css|
+              set.add(self.class.new(css))
+            }
+            set
+          else
+            Set.new(classify(&func).values)
+          end
+        end
+
+        def join(separator=nil)
+          to_a.join(separator)
+        end
+      end
+
+      # sorted_set without rbtree dependency, vendored from
+      # https://github.com/ruby/set/blob/72f08c4/lib/set.rb#L731-L800
+      class SortedSet < Set
+        def initialize(*args)
+          @keys = nil
+          super
+        end
+
+        def clear
+          @keys = nil
+          super
+        end
+
+        def replace(enum)
+          @keys = nil
+          super
+        end
+
+        def add(o)
+          o.respond_to?(:<=>) or raise ArgumentError, "value must respond to <=>"
+          @keys = nil
+          super
+        end
+        alias << add
+
+        def delete(o)
+          @keys = nil
+          @hash.delete(o)
+          self
+        end
+
+        def delete_if
+          block_given? or return enum_for(__method__) { size }
+          n = @hash.size
+          super
+          @keys = nil if @hash.size != n
+          self
+        end
+
+        def keep_if
+          block_given? or return enum_for(__method__) { size }
+          n = @hash.size
+          super
+          @keys = nil if @hash.size != n
+          self
+        end
+
+        def merge(enum)
+          @keys = nil
+          super
+        end
+
+        def each(&block)
+          block or return enum_for(__method__) { size }
+          to_a.each(&block)
+          self
+        end
+
+        def to_a
+          (@keys = @hash.keys).sort! unless @keys
+          @keys.dup
+        end
+
+        def freeze
+          to_a
+          super
+        end
+
+        def rehash
+          @keys = nil
+          super
+        end
+      end
+    end
+  end
+end

data/lib/character_set/ruby_fallback.rb

@@ -1,10 +1,6 @@
-if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
-  require 'sorted_set'
-else
-  require 'set'
-end
 require 'character_set/ruby_fallback/set_methods'
 require 'character_set/ruby_fallback/character_set_methods'
+require 'character_set/ruby_fallback/vendored_set_classes'
 
 class CharacterSet
   module RubyFallback
@@ -16,7 +12,7 @@ class CharacterSet
     end
 
     def initialize(enum = [])
-      @__set = SortedSet.new
+      @__set = CharacterSet::RubyFallback::SortedSet.new
       super
     end
   end

data/lib/character_set/shared_methods.rb

@@ -165,8 +165,8 @@ class CharacterSet
         end
 
         def divide(&func)
-          CharacterSet.require_optional_dependency('set', __method__)
-          Set.new(to_a).divide(&func)
+          require 'character_set/ruby_fallback/vendored_set_classes'
+          CharacterSet::RubyFallback::Set.new(to_a).divide(&func)
         end
       RUBY
 

data/lib/character_set/version.rb

@@ -1,3 +1,3 @@
 class CharacterSet
-  VERSION = '1.6.0'
+  VERSION = '1.7.0'
 end

data/tasks/benchmark.rake

@@ -0,0 +1,20 @@
+desc 'Run all IPS benchmarks'
+task :benchmark do
+  Dir["#{__dir__}/benchmarks/*.rb"].sort.each { |file| load(file) }
+end
+
+namespace :benchmark do
+  desc 'Run all IPS benchmarks and store the comparison results in BENCHMARK.md'
+  task :write_to_file do
+    Rake.application[:benchmark].invoke
+
+    # extract comparison results from reports
+    results = $benchmark_results
+      .map { |caption, report| "```\n#{caption}\n\n#{report[/(?<=Comparison:).+/m].strip}\n```" }
+      .join("\n")
+      .gsub(/ \(±[^)]+\) |(?<=same-ish).*/, '') # remove some noise
+
+    File.write "#{__dir__}/../BENCHMARK.md",
+               "Results of `rake:benchmark` on #{RUBY_DESCRIPTION}\n\n#{results}\n"
+  end
+end

data/tasks/benchmarks/shared.rb

@@ -0,0 +1,28 @@
+require 'benchmark/ips'
+require_relative '../../lib/character_set'
+if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+  require 'sorted_set'
+else
+  require 'set'
+end
+
+def benchmark(caption: nil, cases: {})
+  with_stdouts($stdout, string_io = StringIO.new) do
+    puts caption
+    Benchmark.ips do |x|
+      cases.each { |label, callable| x.report(label, &callable) }
+      x.compare!
+    end
+  end
+  ($benchmark_results ||= {})[caption] = string_io.string
+end
+
+def with_stdouts(*ios)
+  old_stdout = $stdout
+  ios.define_singleton_method(:method_missing) { |*args| each { |io| io.send(*args) } }
+  ios.define_singleton_method(:respond_to?) { |*args| IO.respond_to?(*args) }
+  $stdout = ios
+  yield
+ensure
+  $stdout = old_stdout
+end

data/tasks/sync_casefold_data.rake

@@ -0,0 +1,20 @@
+desc 'Download unicode casefold data and write new C header file'
+task :sync_casefold_data do
+  src_path = './CaseFolding.txt'
+  dst_path = "#{__dir__}/../ext/character_set/unicode_casefold_table.h"
+
+  `wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt`
+
+  mapping = File.foreach(src_path).each_with_object({}) do |line, hash|
+    from, type, to = line.split(/\s*;\s*/).first(3)
+    # type 'C' stands for 'common', excludes mappings to multiple chars
+    hash[from] = to if type == 'C'
+  end.sort
+
+  content = File.read(dst_path + '.tmpl')
+    .sub(/(CASEFOLD_COUNT )0/, "\\1#{mapping.count}")
+    .sub('{}', ['{', mapping.map { |a, b| "{0x#{a},0x#{b}}," }, '}'].join("\n"))
+
+  File.write(dst_path, content)
+  File.unlink(src_path)
+end

data/tasks/sync_predefined_sets.rake

@@ -0,0 +1,9 @@
+desc 'Update codepoint data for predefined sets, based on Onigmo'
+task :sync_predefined_sets do
+  %w[assigned emoji whitespace].each do |prop|
+    require 'regexp_property_values'
+    ranges = RegexpPropertyValues[prop].matched_ranges
+    str = ranges.map { |r| "#{r.min.to_s(16)},#{r.max.to_s(16)}\n" }.join.upcase
+    File.write("#{__dir__}/../lib/character_set/predefined_sets/#{prop}.cps", str, mode: 'w')
+  end
+end

data/tasks/sync_ruby_spec.rake

@@ -0,0 +1,65 @@
+desc 'Download relevant ruby/spec tests, adapt to CharacterSet and its variants'
+task :sync_ruby_spec do
+  require 'fileutils'
+
+  variants = {
+    'CharacterSet'       => "#{__dir__}/../spec/ruby-spec/library/character_set",
+    'CharacterSet::Pure' => "#{__dir__}/../spec/ruby-spec/library/character_set_pure",
+  }
+
+  # download fresh specs from ruby/spec repository
+  variants.each do |_, dir|
+    FileUtils.rm_rf(dir)
+    `svn export https://github.com/ruby/spec/trunk/library/set/sortedset #{dir}`
+  end
+
+  # make copies for each CharacterSet variant
+  base = variants.first[1]
+  variants.each_value { |dir| FileUtils.copy_entry(base, dir) unless dir == base }
+
+  # adapt specs to work with CharacterSet
+  variants.each do |class_name, dir|
+    Dir["#{dir}/**/*.rb"].each do |spec|
+      # ignore some tests that do not apply or are covered otherwise
+      if spec =~ %r{/(classify|divide|flatten|initialize|pretty_print)}
+        File.delete(spec)
+        next
+      end
+
+      adapted_content =
+        File.read(spec).
+        # adapt class name
+        gsub('SortedSet', (spec['/shared/'] ? 'variant' : class_name)).
+        gsub(/(it_behaves_like :[^,\n]+), (:[^,\n]+)/, "\\1, #{class_name}, \\2").
+        # get shared specs from a single shared dir at the parent level
+        gsub(/(require_relative ['"])(shared\/)/, '\1../\2').
+        # make 'mspec' syntax rspec-compatible
+        gsub(/describe (.*), shared.*$/, 'shared_examples \1 do |variant, method|').
+        gsub(/be_(false|true)/, 'be \1').
+        gsub('stub!', 'stub').
+        gsub('mock', 'double').
+        gsub('@method', 'method').
+        # remove unneeded requires
+        gsub(/require 'set'\n/, '').
+        gsub(/require.*spec_helper.*\n/, '').
+        gsub(/\A\n+/, '').
+        # make examples use Integers/codepoints
+        gsub(/1\.0|"cat"|"dog"|"hello"|"test"/, '0').
+        gsub('"one"', '1').
+        gsub('"two"', '2').
+        gsub('"three"', '3').
+        gsub('"four"', '4').
+        gsub('"five"', '5').
+        gsub(/x.(size|length) == 3/, 'x != 3').
+        gsub(/x.(size|length) != 3/, 'x == 3').
+        gsub(/(add)\(\d\)(\.to_a \}.should raise)/, '\1(:foo)\2')
+
+      File.open(spec, 'w') { |f| f.puts adapted_content }
+    end
+  end
+
+  # keep only one copy of the shared specs, at the parent level
+  FileUtils.rm_rf(base + '/../shared')
+  FileUtils.mv(base + '/shared', base + '/../')
+  variants.each_value { |dir| FileUtils.rm_rf(dir + '/shared') }
+end