character_set 1.8.0 → 1.9.0

data/ext/character_set/unicode_casefold_table.h.tmpl

@@ -1,11 +0,0 @@
-// THIS FILE IS GENERATED BY $ rake sync_casefold_data - DO NOT EDIT
-// -*-C-*-
-
-typedef struct casefold_mapping {
-  unsigned long from;
-  unsigned long to;
-} casefold_mapping;
-
-#define CASEFOLD_COUNT 0
-
-static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {};

data/lib/character_set/ruby_fallback/vendored_set_classes.rb

@@ -1,385 +0,0 @@
-# set, vendored from https://github.com/ruby/set/blob/master/lib/set.rb,
-# with comments removed and linted.
-class CharacterSet::RubyFallback::Set
-  Set = self
-  include Enumerable
-
-  def self.[](*ary)
-    new(ary)
-  end
-
-  def initialize(enum = nil, &block)
-    @hash = Hash.new(false)
-
-    enum.nil? and return
-
-    if block
-      do_with_enum(enum) { |o| add(block[o]) }
-    else
-      merge(enum)
-    end
-  end
-
-  def do_with_enum(enum, &block)
-    if enum.respond_to?(:each_entry)
-      enum.each_entry(&block) if block
-    elsif enum.respond_to?(:each)
-      enum.each(&block) if block
-    else
-      raise ArgumentError, "value must be enumerable"
-    end
-  end
-  private :do_with_enum
-
-  def initialize_dup(orig)
-    super
-    @hash = orig.instance_variable_get(:@hash).dup
-  end
-
-  if Kernel.instance_method(:initialize_clone).arity != 1
-    def initialize_clone(orig, **options)
-      super
-      @hash = orig.instance_variable_get(:@hash).clone(**options)
-    end
-  else
-    def initialize_clone(orig)
-      super
-      @hash = orig.instance_variable_get(:@hash).clone
-    end
-  end
-
-  def freeze
-    @hash.freeze
-    super
-  end
-
-  def size
-    @hash.size
-  end
-  alias length size
-
-  def empty?
-    @hash.empty?
-  end
-
-  def clear
-    @hash.clear
-    self
-  end
-
-  def to_a
-    @hash.keys
-  end
-
-  def include?(o)
-    @hash[o]
-  end
-  alias member? include?
-
-  def superset?(set)
-    case
-    when set.instance_of?(self.class) && @hash.respond_to?(:>=)
-      @hash >= set.instance_variable_get(:@hash)
-    when set.is_a?(Set)
-      size >= set.size && set.all? { |o| include?(o) }
-    else
-      raise ArgumentError, "value must be a set"
-    end
-  end
-  alias >= superset?
-
-  def proper_superset?(set)
-    case
-    when set.instance_of?(self.class) && @hash.respond_to?(:>)
-      @hash > set.instance_variable_get(:@hash)
-    when set.is_a?(Set)
-      size > set.size && set.all? { |o| include?(o) }
-    else
-      raise ArgumentError, "value must be a set"
-    end
-  end
-  alias > proper_superset?
-
-  def subset?(set)
-    case
-    when set.instance_of?(self.class) && @hash.respond_to?(:<=)
-      @hash <= set.instance_variable_get(:@hash)
-    when set.is_a?(Set)
-      size <= set.size && all? { |o| set.include?(o) }
-    else
-      raise ArgumentError, "value must be a set"
-    end
-  end
-  alias <= subset?
-
-  def proper_subset?(set)
-    case
-    when set.instance_of?(self.class) && @hash.respond_to?(:<)
-      @hash < set.instance_variable_get(:@hash)
-    when set.is_a?(Set)
-      size < set.size && all? { |o| set.include?(o) }
-    else
-      raise ArgumentError, "value must be a set"
-    end
-  end
-  alias < proper_subset?
-
-  def <=>(set)
-    return unless set.is_a?(Set)
-
-    case size <=> set.size
-    when -1 then -1 if proper_subset?(set)
-    when +1 then +1 if proper_superset?(set)
-    else 0 if self.==(set)
-    end
-  end
-
-  def intersect?(set)
-    case set
-    when Set
-      if size < set.size
-        any? { |o| set.include?(o) }
-      else
-        set.any? { |o| include?(o) }
-      end
-    when Enumerable
-      set.any? { |o| include?(o) }
-    else
-      raise ArgumentError, "value must be enumerable"
-    end
-  end
-
-  def disjoint?(set)
-    !intersect?(set)
-  end
-
-  def each(&block)
-    block_given? or return enum_for(__method__) { size }
-    @hash.each_key(&block)
-    self
-  end
-
-  def add(o)
-    @hash[o] = true
-    self
-  end
-  alias << add
-
-  def add?(o)
-    add(o) unless include?(o)
-  end
-
-  def delete(o)
-    @hash.delete(o)
-    self
-  end
-
-  def delete?(o)
-    delete(o) if include?(o)
-  end
-
-  def delete_if
-    block_given? or return enum_for(__method__) { size }
-    select { |o| yield o }.each { |o| @hash.delete(o) }
-    self
-  end
-
-  def keep_if
-    block_given? or return enum_for(__method__) { size }
-    reject { |o| yield o }.each { |o| @hash.delete(o) }
-    self
-  end
-
-  def reject!(&block)
-    block_given? or return enum_for(__method__) { size }
-    n = size
-    delete_if(&block)
-    self if size != n
-  end
-
-  def select!(&block)
-    block_given? or return enum_for(__method__) { size }
-    n = size
-    keep_if(&block)
-    self if size != n
-  end
-
-  alias filter! select!
-
-  def merge(*enums, **_rest)
-    enums.each do |enum|
-      if enum.instance_of?(self.class)
-        @hash.update(enum.instance_variable_get(:@hash))
-      else
-        do_with_enum(enum) { |o| add(o) }
-      end
-    end
-
-    self
-  end
-
-  def subtract(enum)
-    do_with_enum(enum) { |o| delete(o) }
-    self
-  end
-
-  def |(enum)
-    dup.merge(enum)
-  end
-  alias + |
-  alias union |
-
-  def -(enum)
-    dup.subtract(enum)
-  end
-  alias difference -
-
-  def &(enum)
-    n = self.class.new
-    if enum.is_a?(Set)
-      if enum.size > size
-        each { |o| n.add(o) if enum.include?(o) }
-      else
-        enum.each { |o| n.add(o) if include?(o) }
-      end
-    else
-      do_with_enum(enum) { |o| n.add(o) if include?(o) }
-    end
-    n
-  end
-  alias intersection &
-
-  def ^(enum)
-    n = Set.new(enum)
-    each { |o| n.add(o) unless n.delete?(o) }
-    n
-  end
-
-  def ==(other)
-    if self.equal?(other)
-      true
-    elsif other.instance_of?(self.class)
-      @hash == other.instance_variable_get(:@hash)
-    elsif other.is_a?(Set) && self.size == other.size
-      other.all? { |o| @hash.include?(o) }
-    else
-      false
-    end
-  end
-
-  def hash
-    @hash.hash
-  end
-
-  def eql?(o)
-    return false unless o.is_a?(Set)
-    @hash.eql?(o.instance_variable_get(:@hash))
-  end
-
-  alias === include?
-
-  def classify
-    block_given? or return enum_for(__method__) { size }
-
-    h = {}
-
-    each { |i|
-      (h[yield(i)] ||= self.class.new).add(i)
-    }
-
-    h
-  end
-
-  def divide(&func)
-    func or return enum_for(__method__) { size }
-
-    if func.arity == 2
-      require 'tsort'
-
-      class << dig = {}
-        include TSort
-
-        alias tsort_each_node each_key
-        def tsort_each_child(node, &block)
-          fetch(node).each(&block)
-        end
-      end
-
-      each { |u|
-        dig[u] = a = []
-        each{ |v| func.call(u, v) and a << v }
-      }
-
-      set = Set.new()
-      dig.each_strongly_connected_component { |css|
-        set.add(self.class.new(css))
-      }
-      set
-    else
-      Set.new(classify(&func).values)
-    end
-  end
-end
-
-# sorted_set without rbtree dependency, vendored from
-# https://github.com/ruby/set/blob/72f08c4/lib/set.rb#L731-L800
-class CharacterSet::RubyFallback::SortedSet < CharacterSet::RubyFallback::Set
-  def initialize(*args)
-    @keys = nil
-    super
-  end
-
-  def clear
-    @keys = nil
-    super
-  end
-
-  def add(o)
-    @keys = nil
-    super
-  end
-  alias << add
-
-  def delete(o)
-    @keys = nil
-    @hash.delete(o)
-    self
-  end
-
-  def delete_if
-    block_given? or return enum_for(__method__) { size }
-    n = @hash.size
-    super
-    @keys = nil if @hash.size != n
-    self
-  end
-
-  def keep_if
-    block_given? or return enum_for(__method__) { size }
-    n = @hash.size
-    super
-    @keys = nil if @hash.size != n
-    self
-  end
-
-  def merge(enum)
-    @keys = nil
-    super
-  end
-
-  def each(&block)
-    block or return enum_for(__method__) { size }
-    to_a.each(&block)
-    self
-  end
-
-  def to_a
-    (@keys = @hash.keys).sort! unless @keys
-    @keys.dup
-  end
-
-  def freeze
-    to_a
-    super
-  end
-end

data/tasks/benchmark.rake

@@ -1,20 +0,0 @@
-desc 'Run all IPS benchmarks'
-task :benchmark do
-  Dir["#{__dir__}/benchmarks/*.rb"].sort.each { |file| load(file) }
-end
-
-namespace :benchmark do
-  desc 'Run all IPS benchmarks and store the comparison results in BENCHMARK.md'
-  task :write_to_file do
-    Rake.application[:benchmark].invoke
-
-    # extract comparison results from reports
-    results = $benchmark_results
-      .map { |caption, report| "```\n#{caption}\n\n#{report[/(?<=Comparison:).+/m].strip}\n```" }
-      .join("\n")
-      .gsub(/ \(±[^)]+\) |(?<=same-ish).*/, '') # remove some noise
-
-    File.write "#{__dir__}/../BENCHMARK.md",
-               "Results of `rake:benchmark` on #{RUBY_DESCRIPTION}\n\n#{results}\n"
-  end
-end

data/tasks/benchmarks/count_in.rb

@@ -1,13 +0,0 @@
-require_relative './shared'
-
-str = 'Lorem ipsum et dolorem'
-tr = '^A-Za-z'
-cs = CharacterSet.non_ascii_letter
-
-benchmark(
-  caption: 'Counting non-letters',
-  cases: {
-    'String#count'          => -> { str.count(tr) },
-    'CharacterSet#count_in' => -> { cs.count_in(str) },
-  }
-)

data/tasks/benchmarks/cover.rb

@@ -1,25 +0,0 @@
-require_relative './shared'
-
-str = 'Lorem ipsum et dolorem'
-rx = /\S/
-cs = CharacterSet.whitespace.inversion
-
-benchmark(
-  caption: 'Detecting non-whitespace',
-  cases: {
-    'Regexp#match?'       => -> { rx.match?(str) },
-    'CharacterSet#cover?' => -> { cs.cover?(str) },
-  }
-)
-
-str = 'Lorem ipsum et dolorem'
-rx = /[^a-z]/i
-cs = CharacterSet.new('A'..'Z') + CharacterSet.new('a'..'z')
-
-benchmark(
-  caption: 'Detecting non-letters',
-  cases: {
-    'Regexp#match?'       => -> { rx.match?(str) },
-    'CharacterSet#cover?' => -> { cs.cover?(str) },
-  }
-)

data/tasks/benchmarks/delete_in.rb

@@ -1,29 +0,0 @@
-require_relative './shared'
-
-str = 'Lorem     ipsum       et      dolorem'
-rx = /\s/
-trt = "\t\n\v\f\r\s"
-cs = CharacterSet.whitespace
-
-benchmark(
-  caption: 'Removing ASCII whitespace',
-  cases: {
-    'String#gsub'            => -> { str.gsub(rx, '') },
-    'String#tr'              => -> { str.tr(trt, '') },
-    'CharacterSet#delete_in' => -> { cs.delete_in(str) },
-  }
-)
-
-str = 'Lörem ipsüm ⛷ et dölörem'
-rx = /[\s\p{emoji}äüö]/
-trt = "\t\n\v\f\r\s😀-🙏äüö"
-cs = CharacterSet.whitespace + CharacterSet.emoji + CharacterSet['ä', 'ö', 'ü']
-
-benchmark(
-  caption: 'Removing whitespace, emoji and umlauts',
-  cases: {
-    'String#gsub'            => -> { str.gsub(rx, '') },
-    'String#tr'              => -> { str.tr(trt, '') },
-    'CharacterSet#delete_in' => -> { cs.delete_in(str) },
-  }
-)

data/tasks/benchmarks/keep_in.rb

@@ -1,29 +0,0 @@
-require_relative './shared'
-
-str = 'Lorem ipsum et dolorem'
-rx = /\S/
-trt = "\u{0080}-\u{10FFFF}" # approximation
-cs = CharacterSet.whitespace
-
-benchmark(
-  caption: 'Removing non-whitespace',
-  cases: {
-    'String#gsub'          => -> { str.gsub(rx, '') },
-    'String#tr'            => -> { str.tr(trt, '') },
-    'CharacterSet#keep_in' => -> { cs.keep_in(str) },
-  }
-)
-
-str = 'Lorem ipsum ⛷ et dolorem'
-rx = /\p{^emoji}/
-trt = "\u0000-\u{1F599}\u{1F650}-\u{10FFFF}"
-cs = CharacterSet.emoji
-
-benchmark(
-  caption: 'Keeping only emoji',
-  cases: {
-    'String#gsub'          => -> { str.gsub(rx, '') },
-    'String#tr'            => -> { str.tr(trt, '') },
-    'CharacterSet#keep_in' => -> { cs.keep_in(str) },
-  }
-)

data/tasks/benchmarks/scan.rb

@@ -1,13 +0,0 @@
-require_relative './shared'
-
-str = 'Lorem ipsum ⛷ et dolorem'
-rx = /\p{emoji}/
-cs = CharacterSet.emoji
-
-benchmark(
-  caption: 'Extracting emoji to an Array',
-  cases: {
-    'String#scan'       => -> { str.scan(rx) },
-    'CharacterSet#scan' => -> { cs.scan(str) },
-  }
-)

data/tasks/benchmarks/shared.rb

@@ -1,28 +0,0 @@
-require 'benchmark/ips'
-require_relative '../../lib/character_set'
-if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
-  require 'sorted_set'
-else
-  require 'set'
-end
-
-def benchmark(caption: nil, cases: {})
-  with_stdouts($stdout, string_io = StringIO.new) do
-    puts caption
-    Benchmark.ips do |x|
-      cases.each { |label, callable| x.report(label, &callable) }
-      x.compare!
-    end
-  end
-  ($benchmark_results ||= {})[caption] = string_io.string
-end
-
-def with_stdouts(*ios)
-  old_stdout = $stdout
-  ios.define_singleton_method(:method_missing) { |*args| each { |io| io.send(*args) } }
-  ios.define_singleton_method(:respond_to?) { |*args| IO.respond_to?(*args) }
-  $stdout = ios
-  yield
-ensure
-  $stdout = old_stdout
-end

data/tasks/benchmarks/used_by.rb

@@ -1,25 +0,0 @@
-require_relative './shared'
-
-str = 'Lorem ipsum et dolorem'
-rx = /\s/
-cs = CharacterSet.whitespace
-
-benchmark(
-  caption: 'Detecting whitespace',
-  cases: {
-    'Regexp#match?'         => -> { rx.match?(str) },
-    'CharacterSet#used_by?' => -> { cs.used_by?(str) },
-  }
-)
-
-str = 'Lorem ipsum et dolorem' * 20 + '⛷' + 'Lorem ipsum et dolorem' * 20
-rx = /\p{emoji}/
-cs = CharacterSet.emoji
-
-benchmark(
-  caption: 'Detecting emoji in a large string',
-  cases: {
-    'Regexp#match?'         => -> { rx.match?(str) },
-    'CharacterSet#used_by?' => -> { cs.used_by?(str) },
-  }
-)

data/tasks/benchmarks/z_add.rb

@@ -1,12 +0,0 @@
-require_relative './shared'
-
-cs = CharacterSet[]
-ss = SortedSet[]
-
-benchmark(
-  caption: 'Adding entries',
-  cases: {
-    'CharacterSet#add' => -> { cs.add(rand(0x10FFFF)) },
-    'SortedSet#add'    => -> { ss.add(rand(0x10FFFF)) },
-  }
-)

data/tasks/benchmarks/z_delete.rb

@@ -1,12 +0,0 @@
-require_relative './shared'
-
-cs = CharacterSet.new(0..0x10FFFF)
-ss = SortedSet.new(0..0x10FFFF)
-
-benchmark(
-  caption: 'Removing entries',
-  cases: {
-    'CharacterSet#delete' => -> { cs.delete(rand(0x10FFFF)) },
-    'SortedSet#delete'    => -> { ss.delete(rand(0x10FFFF)) },
-  }
-)

data/tasks/benchmarks/z_merge.rb

@@ -1,15 +0,0 @@
-require_relative './shared'
-
-cs1 = CharacterSet.new(0...0x88000)
-cs2 = CharacterSet.new(0x88000..0x10FFFF)
-
-ss1 = SortedSet.new(0...0x88000)
-ss2 = SortedSet.new(0x88000..0x10FFFF)
-
-benchmark(
-  caption: 'Merging entries',
-  cases: {
-    'CharacterSet#merge' => -> { cs1.merge(cs2) },
-    'SortedSet#merge'    => -> { ss1.merge(ss2) },
-  }
-)

data/tasks/benchmarks/z_minmax.rb

@@ -1,12 +0,0 @@
-require_relative './shared'
-
-cs = CharacterSet.new(0..0xFFFF)
-ss = SortedSet.new(0..0xFFFF)
-
-benchmark(
-  caption: 'Getting the min and max',
-  cases: {
-    'CharacterSet#minmax' => -> { cs.minmax },
-    'SortedSet#minmax'    => -> { ss.minmax },
-  }
-)

data/tasks/sync_casefold_data.rake

@@ -1,20 +0,0 @@
-desc 'Download unicode casefold data and write new C header file'
-task :sync_casefold_data do
-  src_path = './CaseFolding.txt'
-  dst_path = "#{__dir__}/../ext/character_set/unicode_casefold_table.h"
-
-  `wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt`
-
-  mapping = File.foreach(src_path).each_with_object({}) do |line, hash|
-    from, type, to = line.split(/\s*;\s*/).first(3)
-    # type 'C' stands for 'common', excludes mappings to multiple chars
-    hash[from] = to if type == 'C'
-  end.sort
-
-  content = File.read(dst_path + '.tmpl')
-    .sub(/(CASEFOLD_COUNT )0/, "\\1#{mapping.count}")
-    .sub('{}', ['{', mapping.map { |a, b| "{0x#{a},0x#{b}}," }, '}'].join("\n"))
-
-  File.write(dst_path, content)
-  File.unlink(src_path)
-end

data/tasks/sync_predefined_sets.rake

@@ -1,9 +0,0 @@
-desc 'Update codepoint data for predefined sets, based on Onigmo'
-task :sync_predefined_sets do
-  %w[assigned emoji whitespace].each do |prop|
-    require 'regexp_property_values'
-    ranges = RegexpPropertyValues[prop].matched_ranges
-    str = ranges.map { |r| "#{r.min.to_s(16)},#{r.max.to_s(16)}\n" }.join.upcase
-    File.write("#{__dir__}/../lib/character_set/predefined_sets/#{prop}.cps", str, mode: 'w')
-  end
-end