character_set 1.3.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae7ec84b0727a804bf4d82564e6609fdd0bf070fd0e20c0a5688b579e320bc30
-  data.tar.gz: b73dec9fbd4abf83fae5881de89e4e1876e48bcefc3ef935401d5adbeb9c6c8e
+  metadata.gz: e216e6c199ac9443cda9180a9e35d5ed92b50b45c12e7f64f45d74ecd2cf08d6
+  data.tar.gz: 5f3634d426dc33875d6c197ce75466544d97808b1e8b1858ac56d93422b226e8
 SHA512:
-  metadata.gz: 2b84916c89dcd6a234cc5acedfc604f664a9e285c92b3bae6bade748ad3d9c275fb3307fb5721142e52dbedc9b16da65285a8ebd87cd686b55391f222ef1b4f8
-  data.tar.gz: 25147010da0adfd869891d50d51e265c2b4f28e1b0cb70727d9784b11c3944b9a06a9844a2068f529e487028c214f44e2ab60271a9a5730cdd40bb04dd989aaf
+  metadata.gz: d24cfaa40b6e4e472e1f76cc8b6f7f3f1282e6830c0cbf76c4810c0f6f365c7419a19816d0b741cee99eb428dae03fc1d60eecab7d1ba6d210015f0cf2d5ee14
+  data.tar.gz: 2bd7ea63b286e106358293b1428a687374d0cd2cdc985b2da5b5cf1f45c6c541cb0ddde5b06477243cf4011065cfac7fa6bb8a521fb144a750c90039d268f03b

data/.gitattributes

@@ -1,3 +1,3 @@
 *.cps linguist-detectable=false
 benchmarks/* linguist-detectable=false
-spec/ruby-spec/* linguist-vendored
+spec/* linguist-detectable=false

data/.github/workflows/gouteur.yml

@@ -0,0 +1,20 @@
+name: gouteur
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Prepare
+        run: |
+          bundle install --jobs 4
+          bundle exec rake compile
+      - name: Test
+        run: bundle exec gouteur

data/.github/workflows/lint.yml

@@ -0,0 +1,29 @@
+# based on https://github.com/rails/rails/blob/4a78dcb/.github/workflows/rubocop.yml
+
+name: rubocop linting
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Cache gems
+      uses: actions/cache@v1
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-rubocop-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-rubocop-
+    - name: Install gems
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run rubocop
+      run: bundle exec rubocop --lint

data/.github/workflows/tests.yml

@@ -0,0 +1,26 @@
+name: tests
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '11 11 14 * *' # at 11:11 am on the 14th of every month
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby: [ '2.2', '2.7', '3.0', '3.1', 'ruby-head', 'jruby-head' ]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Install dependencies
+        run: bundle install --jobs 4
+      - name: Test with Rake
+        run: bundle exec rake

data/.gitignore

@@ -15,6 +15,7 @@
 .ruby-version
 .tags
 .tags1
+.tool-versions
 .vscode
 bbin/
 binstubs/*

data/.gouteur.yml

@@ -0,0 +1,2 @@
+repos:
+  - uri: https://github.com/jaynetics/js_regex

data/.rubocop.yml

@@ -0,0 +1,17 @@
+AllCops:
+  Exclude:
+    - '**/doc/*'
+    - '**/pkg/*'
+    - '**/spec/ruby-spec/**/*'
+    - '**/vendor/**/*' # vendored dependencies
+  NewCops: enable
+  RubyInterpreters:
+    - ruby
+    - rake
+  TargetRubyVersion: 2.5 # really 2.1, but 2.5 is lowest supported by rubocop
+
+Lint/AmbiguousOperatorPrecedence:
+  Enabled: false
+
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false

data/BENCHMARK.md

@@ -1,86 +1,90 @@
-Results of `rake:benchmark` on ruby 2.6.2p47 (2019-03-13 revision 67232) [x86_64-darwin18]
+Results of `rake:benchmark` on ruby 3.2.0dev (2022-02-14T14:35:54Z master 26187a8520) [arm64-darwin21]
 
 ```
 Counting non-letters
 
-CharacterSet#count_in: 12253693.8 i/s
-        String#count:  1737741.7 i/s - 7.05x  slower
+CharacterSet#count_in: 14794607.9 i/s
+        String#count:  3875939.3 i/s - 3.82x slower
 ```
 ```
 Detecting non-whitespace
 
- CharacterSet#cover?: 14058351.9 i/s
-       Regexp#match?:  7907608.1 i/s - 1.78x  slower
+ CharacterSet#cover?: 17448329.0 i/s
+       Regexp#match?: 13089358.1 i/s - 1.33x slower
 ```
 ```
 Detecting non-letters
 
- CharacterSet#cover?: 13341301.6 i/s
-       Regexp#match?:  5187453.3 i/s - 2.57x  slower
+ CharacterSet#cover?: 17565596.9 i/s
+       Regexp#match?:  7951108.0 i/s - 2.21x slower
 ```
 ```
-Removing whitespace
+Removing ASCII whitespace
 
-CharacterSet#delete_in:  2523184.0 i/s
-         String#gsub:   225804.7 i/s - 11.17x  slower
+CharacterSet#delete_in:  6306078.2 i/s
+           String#tr:  4734401.0 i/s - 1.33x slower
+         String#gsub:   211631.8 i/s - 29.80x slower
 ```
 ```
 Removing whitespace, emoji and umlauts
 
-CharacterSet#delete_in:  1712208.6 i/s
-         String#gsub:   278508.8 i/s - 6.15x  slower
+CharacterSet#delete_in:  5984149.6 i/s
+           String#tr:   363643.1 i/s - 16.46x slower
+         String#gsub:   317201.7 i/s - 18.87x slower
 ```
 ```
 Removing non-whitespace
 
-CharacterSet#keep_in:  2760158.1 i/s
-         String#gsub:   232797.7 i/s - 11.86x  slower
+CharacterSet#keep_in:  7650925.6 i/s
+         String#gsub:   207374.6 i/s - 36.89x slower
+           String#tr:       12.3 i/s - 619745.60x slower
 ```
 ```
-Extracting emoji
+Keeping only emoji
 
-CharacterSet#keep_in:  1775758.8 i/s
-         String#gsub:   217649.9 i/s - 8.16x  slower
+CharacterSet#keep_in:  7272940.1 i/s
+         String#gsub:   177993.8 i/s - 40.86x slower
+           String#tr:       12.3 i/s - 590222.71x slower
 ```
 ```
 Extracting emoji to an Array
 
-   CharacterSet#scan:  2579030.8 i/s
-         String#scan:   545107.0 i/s - 4.73x  slower
+   CharacterSet#scan:  2978285.0 i/s
+         String#scan:   865793.8 i/s - 3.44x slower
 ```
 ```
 Detecting whitespace
 
-CharacterSet#used_by?: 13847689.0 i/s
-       Regexp#match?:  7533275.2 i/s - 1.84x  slower
+CharacterSet#used_by?: 17292338.4 i/s
+       Regexp#match?: 11705563.9 i/s - 1.48x slower
 ```
 ```
 Detecting emoji in a large string
 
-CharacterSet#used_by?:   246527.7 i/s
-       Regexp#match?:    92956.5 i/s - 2.65x  slower
+CharacterSet#used_by?:   340444.1 i/s
+       Regexp#match?:   180549.8 i/s - 1.89x slower
 ```
 ```
 Adding entries
 
-    CharacterSet#add:  3102081.7 i/s
-       SortedSet#add:  1897464.8 i/s - 1.63x  slower
+    CharacterSet#add:  4951781.4 i/s
+       SortedSet#add:  1019637.9 i/s - 4.86x slower
 ```
 ```
 Removing entries
 
- CharacterSet#delete:  3240924.1 i/s
-    SortedSet#delete:  2887493.9 i/s - 1.12x  slower
+ CharacterSet#delete:  5006337.6 i/s
+    SortedSet#delete:  3922752.2 i/s - same-ish
 ```
 ```
 Merging entries
 
-  CharacterSet#merge:      536.8 i/s
-     SortedSet#merge:       12.5 i/s - 42.78x  slower
+  CharacterSet#merge:      661.8 i/s
+     SortedSet#merge:        3.9 i/s - 167.82x slower
 ```
 ```
 Getting the min and max
 
- CharacterSet#minmax:  4111960.8 i/s
-    SortedSet#minmax:      756.4 i/s - 5436.39x  slower
+ CharacterSet#minmax:  1212462.2 i/s
+    SortedSet#minmax:      844.4 i/s - 1435.93x slower
 ```

data/CHANGELOG.md

@@ -4,7 +4,56 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## UNRELEASED
+## [1.6.0] - 2022-02-16
+
+### Added
+
+- `::of` now supports both `String` and `Regexp` arguments
+
+### Fixed
+
+- fixed segfault during `String` manipulation on Ruby 3.2.0-dev
+- improved performance for `String` manipulation
+- allow usage in Ractors
+  - predefined sets must be pre-initialized for this, though
+  - e.g. `CharacterSet.ascii`, `keep_character_set(:ascii)` etc.
+  - call them once in the main Ractor to trigger initialization
+
+## [1.5.0] - 2021-12-05
+
+### Added
+
+- new codepoints for `::assigned` and `::emoji` predefined sets, as in Ruby 3.1.0
+- latest unicode case-folding data (for `#case_insensitive`)
+- support for passing any Enumerable to `#disjoint?`, `#intersect?`
+  - this matches recent broadening of these methods in `ruby/set`
+- new instance method `#secure_token` (see README)
+- class method `::of` now accepts more than one `String`
+- `CharacterSet::ExpressionConverter` can now build output of any Set-like class
+
+### Fixed
+
+- `CharacterSet::Pure::of_expression` now returns a `CharacterSet::Pure`
+  - it used to return a regular `CharacterSet`
+
+## [1.4.1] - 2020-01-10
+
+### Fixed
+- multiple fixes for Ruby 3
+  - fixed segfault for some `String` manipulation cases
+  - added `sorted_set` as dependency, so `CharacterSet::Pure` (non-C fallback) works
+- fixed error when parsing a `Regexp` with an empty intersection (e.g. `/[a&&]/`)
+
+## [1.4.0] - 2019-06-07
+
+### Added
+- `#to_s_with_surrogate_ranges` / `Writer::write_surrogate_ranges`
+  - allows for much shorter astral plane representations e.g. in JavaScript
+  - thanks to https://github.com/singpolyma for the suggestion and groundwork (#1)
+- improved performance for `#to_s` / `Writer` by avoiding bugged `Range#minmax`
+
+### Fixed
+- '/' is now escaped by default when stringifying so as to work with //-regexp syntax
 
 ## [1.3.0] - 2019-04-26
 

data/Gemfile

@@ -4,3 +4,17 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in character_set.gemspec
 gemspec
+
+gem 'benchmark-ips', '~> 2.7'
+gem 'get_process_mem', '~> 0.2.3'
+gem 'rake', '~> 13.0'
+gem 'rake-compiler', '~> 1.1'
+gem 'range_compressor', '~> 1.0'
+gem 'regexp_parser', '~> 2.1'
+gem 'regexp_property_values', '~> 1.0'
+gem 'rspec', '~> 3.8'
+if RUBY_VERSION.to_f >= 2.7
+  gem 'codecov', '~> 0.2.12'
+  gem 'gouteur', '~> 1.0.0'
+  gem 'rubocop', '~> 1.8'
+end

data/README.md

@@ -1,18 +1,21 @@
 # CharacterSet
 
 [![Gem Version](https://badge.fury.io/rb/character_set.svg)](http://badge.fury.io/rb/character_set)
-[![Build Status](https://travis-ci.org/jaynetics/character_set.svg?branch=master)](https://travis-ci.org/jaynetics/character_set)
+[![Build Status](https://github.com/jaynetics/character_set/workflows/tests/badge.svg)](https://github.com/jaynetics/character_set/actions)
+[![Build Status](https://github.com/jaynetics/character_set/workflows/gouteur/badge.svg)](https://github.com/jaynetics/character_set/actions)
 [![codecov](https://codecov.io/gh/jaynetics/character_set/branch/master/graph/badge.svg)](https://codecov.io/gh/jaynetics/character_set)
 
-This is a C-extended Ruby gem to work with sets of Unicode codepoints. It can read and write these sets in various formats and implements the stdlib `Set` interface for them.
+This is a C-extended Ruby gem to work with sets of Unicode codepoints.
 
-It also offers an alternate paradigm of `String` processing which grants much better performance than `Regexp` and `String` methods from the stdlib where applicable (see [benchmarks](./BENCHMARK.md)).
+It can [read](#parseinitialize) and [write](#write) sets of codepoints in various formats and it implements the stdlib `Set` interface for them.
+
+It also offers a [way of scrubbing and scanning characters in Strings](#interact-with-strings) that is more semantic and consistently offers better performance than `Regexp` and `String` methods from the stdlib for this (see [benchmarks](./BENCHMARK.md)).
 
 Many parts can be used independently, e.g.:
 - `CharacterSet::Character`
+- `CharacterSet::ExpressionConverter`
 - `CharacterSet::Parser`
 - `CharacterSet::Writer`
-- [`RangeCompressor`](https://github.com/jaynetics/range_compressor)
 
 ## Usage
 
@@ -40,9 +43,10 @@ CharacterSet.parse('[a-c]')
 CharacterSet.parse('\U00000061-\U00000063')
 ```
 
-If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/jaynetics/regexp_property_values) are installed, `::of_regexp` and `::of_property` can also be used. `::of_regexp` can handle intersections, negations, and set nesting.
+If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/jaynetics/regexp_property_values) are installed, `Regexp` and unicode property names can also be read. Regexp intersections, negations, and set nesting are covered, but the `i`-flag is ignored; call `#case_insensitive` on the result if needed.
 
 ```ruby
+CharacterSet.of(/./) # => #<CharacterSet (size: 1112064)>
 CharacterSet.of_property('Thai') # => #<CharacterSet (size: 86)>
 
 require 'character_set/core_ext/regexp_ext'
@@ -143,6 +147,7 @@ CharacterSet['1', 'A'].case_insensitive # => CharacterSet['1', 'A', 'a']
 ```
 
 ### Write
+
 ```ruby
 set = CharacterSet['a', 'b', 'c', 'j', '-']
 
@@ -167,11 +172,32 @@ set.to_s(escape_all: true) { |c| "<#{c.hex}>" } # => "<61>-<63><258><1F929>"
 # disable abbreviation (grouping of codepoints in ranges)
 set.to_s(abbreviate: false) # => "abc\u0258\u{1F929}"
 
-# for full js regex compatibility in case of astral members:
-set.to_s_with_surrogate_alternation # => '(?:[a-c\u0258]|\ud83e\udd29)'
+# astral members require some trickery if we want to target environments
+# that are based on UTF-16 or "UCS-2 with surrogates", such as JavaScript.
+set = CharacterSet['a', 'b', '🤩', '🤪', '🤫']
+
+# Use #to_s_with_surrogate_ranges e.g. for JavaScript:
+set.to_s_with_surrogate_ranges
+# => '(?:[ab]|\uD83E[\uDD29-\uDD2B])'
+
+# Or use #to_s_with_surrogate_alternation if such surrogate set pairs
+# don't work in your target environment:
+set.to_s_with_surrogate_alternation
+# => '(?:[ab]|\uD83E\uDD29|\uD83E\uDD2A|\uD83E\uDD2B)'
+```
+
+### Other features
+
+#### Secure tokens
+
+Generate secure random strings of characters from a set:
+
+```ruby
+CharacterSet.new('a'..'z').secure_token(8) # => "ugwpujmt"
+CharacterSet.crypt.secure_token # => "8.1w7aBT737/pMfcMoO4y2y8/=0xtmo:"
 ```
 
-### Unicode plane methods
+#### Unicode planes
 
 There are some methods to check for planes and to handle ASCII, [BMP](https://en.wikipedia.org/wiki/Plane_%28Unicode%29#Basic_Multilingual_Plane) and astral parts:
 ```Ruby
@@ -188,6 +214,6 @@ CharacterSet['a', 'ü', '🤩'].member_in_plane?(7) # => false
 CharacterSet::Character.new('a').plane # => 0
 ```
 
-### Contributions
+## Contributions
 
 Feel free to send suggestions, point out issues, or submit pull requests.

data/Rakefile

@@ -126,7 +126,7 @@ task :sync_predefined_sets do
   %w[assigned emoji whitespace].each do |prop|
     require 'regexp_property_values'
     ranges = RegexpPropertyValues[prop].matched_ranges
-    str = ranges.map { |r| r.minmax.map { |n| n.to_s(16) }.join(',').upcase + "\n" }.join
+    str = ranges.map { |r| "#{r.min.to_s(16)},#{r.max.to_s(16)}\n" }.join.upcase
     File.write("./lib/character_set/predefined_sets/#{prop}.cps", str, mode: 'w')
   end
 end
@@ -147,8 +147,11 @@ namespace :benchmark do
       f.puts "Results of `rake:benchmark` on #{RUBY_DESCRIPTION}", ''
 
       $store_comparison_results.each do |caption, result|
-        f.puts '```', caption, '',
-               result.strip.gsub(/(same-ish).*$/, '\1').lines[1..-1], '```'
+        f.puts '```',
+               caption,
+               '',
+               result.strip.gsub(/ \(±[^)]+\) /, '').gsub(/(same-ish).*$/, '\1').lines[1..-1],
+               '```'
       end
     end
   end

data/benchmarks/delete_in.rb

@@ -2,24 +2,28 @@ require_relative './shared'
 
 str = 'Lorem     ipsum       et      dolorem'
 rx = /\s/
+trt = "\t\n\v\f\r\s"
 cs = CharacterSet.whitespace
 
 benchmark(
-  caption: 'Removing whitespace',
+  caption: 'Removing ASCII whitespace',
   cases: {
     'String#gsub'            => -> { str.gsub(rx, '') },
+    'String#tr'              => -> { str.tr(trt, '') },
     'CharacterSet#delete_in' => -> { cs.delete_in(str) },
   }
 )
 
 str = 'Lörem ipsüm ⛷ et dölörem'
 rx = /[\s\p{emoji}äüö]/
+trt = "\t\n\v\f\r\s😀-🙏äüö"
 cs = CharacterSet.whitespace + CharacterSet.emoji + CharacterSet['ä', 'ö', 'ü']
 
 benchmark(
   caption: 'Removing whitespace, emoji and umlauts',
   cases: {
     'String#gsub'            => -> { str.gsub(rx, '') },
+    'String#tr'              => -> { str.tr(trt, '') },
     'CharacterSet#delete_in' => -> { cs.delete_in(str) },
   }
 )

data/benchmarks/keep_in.rb

@@ -2,24 +2,28 @@ require_relative './shared'
 
 str = 'Lorem ipsum et dolorem'
 rx = /\S/
+trt = "\u{0080}-\u{10FFFF}" # approximation
 cs = CharacterSet.whitespace
 
 benchmark(
   caption: 'Removing non-whitespace',
   cases: {
     'String#gsub'          => -> { str.gsub(rx, '') },
+    'String#tr'            => -> { str.tr(trt, '') },
     'CharacterSet#keep_in' => -> { cs.keep_in(str) },
   }
 )
 
 str = 'Lorem ipsum ⛷ et dolorem'
 rx = /\p{^emoji}/
+trt = "\u0000-\u{1F599}\u{1F650}-\u{10FFFF}"
 cs = CharacterSet.emoji
 
 benchmark(
-  caption: 'Extracting emoji',
+  caption: 'Keeping only emoji',
   cases: {
     'String#gsub'          => -> { str.gsub(rx, '') },
+    'String#tr'            => -> { str.tr(trt, '') },
     'CharacterSet#keep_in' => -> { cs.keep_in(str) },
   }
 )

data/benchmarks/shared.rb

@@ -3,7 +3,11 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'benchmark/ips'
 require 'character_set'
-require 'set'
+if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+  require 'sorted_set'
+else
+  require 'set'
+end
 
 def benchmark(caption: nil, cases: {})
   puts caption

data/character_set.gemspec

@@ -22,13 +22,10 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = '>= 2.1.0'
 
-  s.add_development_dependency 'benchmark-ips', '~> 2.7'
-  s.add_development_dependency 'codecov', '~> 0.1'
-  s.add_development_dependency 'get_process_mem', '~> 0.2.3'
-  s.add_development_dependency 'rake', '~> 12.0'
-  s.add_development_dependency 'rake-compiler', '~> 1.0'
-  s.add_development_dependency 'range_compressor', '~> 1.0'
-  s.add_development_dependency 'regexp_parser', '~> 1.3'
-  s.add_development_dependency 'regexp_property_values', '~> 0.3.5'
-  s.add_development_dependency 'rspec', '~> 3.8'
+  # SortedSet, needed for RubyFallback, was moved to a gem in Ruby 3.
+  # This dependency is only used if the C extension is unavailable.
+  # JRuby has it in the stdlib.
+  if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+    s.add_dependency 'sorted_set', '~> 1.0'
+  end
 end