character_set 1.6.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e216e6c199ac9443cda9180a9e35d5ed92b50b45c12e7f64f45d74ecd2cf08d6
-  data.tar.gz: 5f3634d426dc33875d6c197ce75466544d97808b1e8b1858ac56d93422b226e8
+  metadata.gz: ebb6792f685df02534f1ef04a92d7f0c5fdcb482e5aaa4856d7a39726e17f007
+  data.tar.gz: c6630aab9b6506c46a970ba83c257cd753f8f76760b6ce8d2639f51efba83eeb
 SHA512:
-  metadata.gz: d24cfaa40b6e4e472e1f76cc8b6f7f3f1282e6830c0cbf76c4810c0f6f365c7419a19816d0b741cee99eb428dae03fc1d60eecab7d1ba6d210015f0cf2d5ee14
-  data.tar.gz: 2bd7ea63b286e106358293b1428a687374d0cd2cdc985b2da5b5cf1f45c6c541cb0ddde5b06477243cf4011065cfac7fa6bb8a521fb144a750c90039d268f03b
+  metadata.gz: 4c773a0546d05939d0b295e50355c6efe870a1ed74901d63c24097ff598d4a43bcd00ce2d03fb492a48fd9c03968a79ee78b789d92836843d6621dca3e8f313c
+  data.tar.gz: 560d3c3aa3f7e4daac3b6d2c89fb9dd6840777fa4d5896fb33564023ef745d81a7e4d0e51fe0ba42f6cd4504bc0b088657cd4ef1ab15d213aa1bb096ba404542

data/.github/workflows/gouteur.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: 3.3
       - name: Prepare
         run: |
           bundle install --jobs 4

data/.github/workflows/lint.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.7
+        ruby-version: 3.3
     - name: Cache gems
       uses: actions/cache@v1
       with:

data/.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        ruby: [ '2.2', '2.7', '3.0', '3.1', 'ruby-head', 'jruby-head' ]
+        ruby: [ '2.4', '2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-head' ]
 
     steps:
       - uses: actions/checkout@v2
@@ -24,3 +24,5 @@ jobs:
         run: bundle install --jobs 4
       - name: Test with Rake
         run: bundle exec rake
+      - uses: codecov/codecov-action@v3
+        if: matrix.ruby == '3.2'

data/.rubocop.yml

@@ -15,3 +15,6 @@ Lint/AmbiguousOperatorPrecedence:
 
 Lint/AmbiguousRegexpLiteral:
   Enabled: false
+
+Metrics:
+  Enabled: false

data/BENCHMARK.md

@@ -3,88 +3,88 @@ Results of `rake:benchmark` on ruby 3.2.0dev (2022-02-14T14:35:54Z master 26187a
 ```
 Counting non-letters
 
-CharacterSet#count_in: 14794607.9 i/s
-        String#count:  3875939.3 i/s - 3.82x slower
+CharacterSet#count_in: 14627506.2 i/s
+        String#count:  3859777.0 i/s - 3.79x slower
 ```
 ```
 Detecting non-whitespace
 
- CharacterSet#cover?: 17448329.0 i/s
-       Regexp#match?: 13089358.1 i/s - 1.33x slower
+ CharacterSet#cover?: 17241902.8 i/s
+       Regexp#match?: 12971122.6 i/s - 1.33x slower
 ```
 ```
 Detecting non-letters
 
- CharacterSet#cover?: 17565596.9 i/s
-       Regexp#match?:  7951108.0 i/s - 2.21x slower
+ CharacterSet#cover?: 17243472.3 i/s
+       Regexp#match?:  7957626.9 i/s - 2.17x slower
 ```
 ```
 Removing ASCII whitespace
 
-CharacterSet#delete_in:  6306078.2 i/s
-           String#tr:  4734401.0 i/s - 1.33x slower
-         String#gsub:   211631.8 i/s - 29.80x slower
+CharacterSet#delete_in:  6190975.7 i/s
+           String#tr:  4722716.6 i/s - 1.31x slower
+         String#gsub:   214239.5 i/s - 28.90x slower
 ```
 ```
 Removing whitespace, emoji and umlauts
 
-CharacterSet#delete_in:  5984149.6 i/s
-           String#tr:   363643.1 i/s - 16.46x slower
-         String#gsub:   317201.7 i/s - 18.87x slower
+CharacterSet#delete_in:  5890471.8 i/s
+           String#tr:   348506.8 i/s - 16.90x slower
+         String#gsub:   318268.3 i/s - 18.51x slower
 ```
 ```
 Removing non-whitespace
 
-CharacterSet#keep_in:  7650925.6 i/s
-         String#gsub:   207374.6 i/s - 36.89x slower
-           String#tr:       12.3 i/s - 619745.60x slower
+CharacterSet#keep_in:  7396898.0 i/s
+         String#gsub:   208809.7 i/s - 35.42x slower
+           String#tr:       13.1 i/s - 564682.50x slower
 ```
 ```
 Keeping only emoji
 
-CharacterSet#keep_in:  7272940.1 i/s
-         String#gsub:   177993.8 i/s - 40.86x slower
-           String#tr:       12.3 i/s - 590222.71x slower
+CharacterSet#keep_in:  7022741.1 i/s
+         String#gsub:   180939.6 i/s - 38.81x slower
+           String#tr:       13.1 i/s - 536724.50x slower
 ```
 ```
 Extracting emoji to an Array
 
-   CharacterSet#scan:  2978285.0 i/s
-         String#scan:   865793.8 i/s - 3.44x slower
+   CharacterSet#scan:  3023176.8 i/s
+         String#scan:   893225.8 i/s - 3.38x slower
 ```
 ```
 Detecting whitespace
 
-CharacterSet#used_by?: 17292338.4 i/s
-       Regexp#match?: 11705563.9 i/s - 1.48x slower
+CharacterSet#used_by?: 17284025.9 i/s
+       Regexp#match?: 11847064.5 i/s - 1.46x slower
 ```
 ```
 Detecting emoji in a large string
 
-CharacterSet#used_by?:   340444.1 i/s
-       Regexp#match?:   180549.8 i/s - 1.89x slower
+CharacterSet#used_by?:   341386.1 i/s
+       Regexp#match?:   183121.6 i/s - 1.86x slower
 ```
 ```
 Adding entries
 
-    CharacterSet#add:  4951781.4 i/s
-       SortedSet#add:  1019637.9 i/s - 4.86x slower
+    CharacterSet#add:  4989762.3 i/s
+       SortedSet#add:  1157911.7 i/s - 4.31x slower
 ```
 ```
 Removing entries
 
- CharacterSet#delete:  5006337.6 i/s
-    SortedSet#delete:  3922752.2 i/s - same-ish
+ CharacterSet#delete:  4996703.6 i/s
+    SortedSet#delete:  4177401.5 i/s - same-ish
 ```
 ```
 Merging entries
 
-  CharacterSet#merge:      661.8 i/s
-     SortedSet#merge:        3.9 i/s - 167.82x slower
+  CharacterSet#merge:      666.7 i/s
+     SortedSet#merge:        4.0 i/s - 167.84x slower
 ```
 ```
 Getting the min and max
 
- CharacterSet#minmax:  1212462.2 i/s
-    SortedSet#minmax:      844.4 i/s - 1435.93x slower
+ CharacterSet#minmax:  1596470.9 i/s
+    SortedSet#minmax:      866.4 i/s - 1842.74x slower
 ```

data/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [1.8.0] - 2024-01-07
+
+### Added
+
+- support for `#<=>` and `#join`, which were added to `set` in the meantime
+- support for getting the (overall) character set of a Regexp with multiple expressions
+- support for global and local case-insensitivity in Regexp inputs
+- `Regexp#{covered_by_character_set?,uses_character_set?}` methods (if core ext is used)
+
+## [1.7.0] - 2023-05-12
+
+### Added
+
+- new codepoints for `::assigned` and `::emoji` predefined sets, as in Ruby 3.2.0
+
+### Fixed
+
+- fixed processing of Strings that are not ASCII- or UTF8-encoded
+- removed dependency on `set` and `sorted_set`
+  - thanks to https://github.com/mikebaldry for reporting a related issue (#2)
+
 ## [1.6.0] - 2022-02-16
 
 ### Added
@@ -63,7 +86,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - reduced memory consumption by > 90% for most use cases via dynamic resizing
   - before, every set instance required 136 KB for codepoints
   - now, 16 bytes for a CharacterSet in ASCII space, 8 KB for one in BMP space etc.
-- `#count_in` and `#scan_in` methods for `String` interaction
+- `#count_in` and `#scan` methods for `String` interaction
 - new predefined sets `::any`/`::all`, `::assigned`, `::surrogate`
 - conversion methods `#assigned_part`, `#valid_part`
 - sectioning methods `#ascii_part`, `#plane(n)`

data/Gemfile

@@ -7,14 +7,15 @@ gemspec
 
 gem 'benchmark-ips', '~> 2.7'
 gem 'get_process_mem', '~> 0.2.3'
-gem 'rake', '~> 13.0'
+gem 'rake', '~> 13.1'
 gem 'rake-compiler', '~> 1.1'
 gem 'range_compressor', '~> 1.0'
-gem 'regexp_parser', '~> 2.1'
-gem 'regexp_property_values', '~> 1.0'
+gem 'regexp_parser', '~> 2.9'
+gem 'regexp_property_values', '~> 1.5'
 gem 'rspec', '~> 3.8'
-if RUBY_VERSION.to_f >= 2.7
-  gem 'codecov', '~> 0.2.12'
+gem 'warning', '~> 1.3'
+if RUBY_VERSION.to_f >= 3.0
   gem 'gouteur', '~> 1.0.0'
-  gem 'rubocop', '~> 1.8'
+  gem 'rubocop', '~> 1.59'
+  gem 'simplecov-cobertura', require: false
 end

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Janosch Müller
+Copyright (c) 2018-2023 Janosch Müller
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -3,7 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/character_set.svg)](http://badge.fury.io/rb/character_set)
 [![Build Status](https://github.com/jaynetics/character_set/workflows/tests/badge.svg)](https://github.com/jaynetics/character_set/actions)
 [![Build Status](https://github.com/jaynetics/character_set/workflows/gouteur/badge.svg)](https://github.com/jaynetics/character_set/actions)
-[![codecov](https://codecov.io/gh/jaynetics/character_set/branch/master/graph/badge.svg)](https://codecov.io/gh/jaynetics/character_set)
+[![Coverage](https://codecov.io/gh/jaynetics/character_set/branch/main/graph/badge.svg?token=oY7gcWNbIN)](https://codecov.io/gh/jaynetics/character_set)
 
 This is a C-extended Ruby gem to work with sets of Unicode codepoints.
 
@@ -43,7 +43,7 @@ CharacterSet.parse('[a-c]')
 CharacterSet.parse('\U00000061-\U00000063')
 ```
 
-If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/jaynetics/regexp_property_values) are installed, `Regexp` and unicode property names can also be read. Regexp intersections, negations, and set nesting are covered, but the `i`-flag is ignored; call `#case_insensitive` on the result if needed.
+If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/jaynetics/regexp_property_values) are installed, `Regexp` instances and unicode property names can also be read.
 
 ```ruby
 CharacterSet.of(/./) # => #<CharacterSet (size: 1112064)>
@@ -96,7 +96,7 @@ string # => ''
 
 ```ruby
 CharacterSet.non_ascii.count_in('Tüür') # => 2
-CharacterSet.non_ascii.scan_in('Tüür') # => ['ü', 'ü']
+CharacterSet.non_ascii.scan('Tüür') # => ['ü', 'ü']
 ```
 
 There is also a core extension for String interaction.

data/Rakefile

@@ -3,6 +3,8 @@ require 'rspec/core/rake_task'
 require 'rubygems/package_task'
 require 'rake/extensiontask'
 
+Dir['tasks/**/*.rake'].each { |file| load(file) }
+
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
@@ -34,129 +36,6 @@ end
 
 task package: 'java:gem'
 
-desc 'Download relevant ruby/spec tests, adapt to CharacterSet and its variants'
-task :sync_ruby_spec do
-  require 'fileutils'
-
-  variants = {
-    'CharacterSet'       => './spec/ruby-spec/library/character_set',
-    'CharacterSet::Pure' => './spec/ruby-spec/library/character_set_pure',
-  }
-
-  # download fresh specs from ruby/spec repository
-  variants.each do |_, dir|
-    FileUtils.rm_rf(dir) if File.exist?(dir)
-    `svn export https://github.com/ruby/spec/trunk/library/set/sortedset #{dir}`
-  end
-
-  # make copies for each CharacterSet variant
-  base = variants.first[1]
-  variants.each_value { |dir| FileUtils.copy_entry(base, dir) unless dir == base }
-
-  # adapt specs to work with CharacterSet
-  variants.each do |class_name, dir|
-    Dir["#{dir}/**/*.rb"].each do |spec|
-      # ignore some tests that do not apply or are covered otherwise
-      if spec =~ %r{/(classify|divide|flatten|initialize|pretty_print)}
-        File.delete(spec)
-        next
-      end
-
-      adapted_content =
-        File.read(spec).
-        # adapt class name
-        gsub('SortedSet', (spec['/shared/'] ? 'variant' : class_name)).
-        gsub(/(it_behaves_like :[^,\n]+), (:[^,\n]+)/, "\\1, #{class_name}, \\2").
-        # get shared specs from a single shared dir at the parent level
-        gsub(/(require_relative ['"])(shared\/)/, '\1../\2').
-        # make 'mspec' syntax rspec-compatible
-        gsub(/describe (.*), shared.*$/, 'shared_examples \1 do |variant, method|').
-        gsub(/be_(false|true)/, 'be \1').
-        gsub('stub!', 'stub').
-        gsub('mock', 'double').
-        gsub('@method', 'method').
-        # remove unneeded requires
-        gsub(/require 'set'\n/, '').
-        gsub(/require.*spec_helper.*\n/, '').
-        gsub(/\A\n+/, '').
-        # make examples use Integers/codepoints
-        gsub(/1\.0|"cat"|"dog"|"hello"|"test"/, '0').
-        gsub('"one"', '1').
-        gsub('"two"', '2').
-        gsub('"three"', '3').
-        gsub('"four"', '4').
-        gsub('"five"', '5').
-        gsub(/x.(size|length) == 3/, 'x != 3').
-        gsub(/x.(size|length) != 3/, 'x == 3').
-        gsub(/(add)\(\d\)(\.to_a \}.should raise)/, '\1(:foo)\2')
-
-      File.open(spec, 'w') { |f| f.puts adapted_content }
-    end
-  end
-
-  # keep only one copy of the shared specs, at the parent level
-  FileUtils.rm_rf(base + '/../shared')
-  FileUtils.mv(base + '/shared', base + '/../')
-  variants.each_value { |dir| FileUtils.rm_rf(dir + '/shared') }
-end
-
-desc 'Download unicode casefold data and write new C header file'
-task :sync_casefold_data do
-  src_path = './CaseFolding.txt'
-  dst_path = './ext/character_set/unicode_casefold_table.h'
-
-  `wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt`
-
-  mapping = File.foreach(src_path).each_with_object({}) do |line, hash|
-    from, type, to = line.split(/\s*;\s*/).first(3)
-    # type 'C' stands for 'common', excludes mappings to multiple chars
-    hash[from] = to if type == 'C'
-  end.sort
-
-  content = File.read(dst_path + '.tmpl')
-    .sub(/(CASEFOLD_COUNT )0/, "\\1#{mapping.count}")
-    .sub('{}', ['{', mapping.map { |a, b| "{0x#{a},0x#{b}}," }, '}'].join("\n"))
-
-  File.write(dst_path, content)
-  File.unlink(src_path)
-end
-
-desc 'Update codepoint data for predefined sets, based on Onigmo'
-task :sync_predefined_sets do
-  %w[assigned emoji whitespace].each do |prop|
-    require 'regexp_property_values'
-    ranges = RegexpPropertyValues[prop].matched_ranges
-    str = ranges.map { |r| "#{r.min.to_s(16)},#{r.max.to_s(16)}\n" }.join.upcase
-    File.write("./lib/character_set/predefined_sets/#{prop}.cps", str, mode: 'w')
-  end
-end
-
-desc 'Run all IPS benchmarks'
-task :benchmark do
-  Dir['./benchmarks/*.rb'].sort.each { |file| require file }
-end
-
-namespace :benchmark do
-  desc 'Run all IPS benchmarks and store the comparison results in BENCHMARK.md'
-  task :write_to_file do
-    $store_comparison_results = {}
-
-    Rake.application[:benchmark].invoke
-
-    File.open('BENCHMARK.md', 'w') do |f|
-      f.puts "Results of `rake:benchmark` on #{RUBY_DESCRIPTION}", ''
-
-      $store_comparison_results.each do |caption, result|
-        f.puts '```',
-               caption,
-               '',
-               result.strip.gsub(/ \(±[^)]+\) /, '').gsub(/(same-ish).*$/, '\1').lines[1..-1],
-               '```'
-      end
-    end
-  end
-end
-
 unless RUBY_PLATFORM =~ /java/
   # recompile before benchmarking or running specs
   task(:benchmark).enhance([:compile])

data/character_set.gemspec

@@ -21,11 +21,4 @@ Gem::Specification.new do |s|
   s.extensions  = %w[ext/character_set/extconf.rb]
 
   s.required_ruby_version = '>= 2.1.0'
-
-  # SortedSet, needed for RubyFallback, was moved to a gem in Ruby 3.
-  # This dependency is only used if the C extension is unavailable.
-  # JRuby has it in the stdlib.
-  if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
-    s.add_dependency 'sorted_set', '~> 1.0'
-  end
 end