character_set 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d2e4067480e00d5d03db2bbd1ee4f222f936e0f2
+  data.tar.gz: 0e4c0bc6cf393b1a81dc368ee86f94d0dea10a82
+SHA512:
+  metadata.gz: d9150168393512190a496ed10af91a1eaa49eb2a01d3fb623de9586eb4fbd354dfea172bf6174ab180f6620ae6ca13a01f94ec26a95fbf118f48f611b4d7acd7
+  data.tar.gz: cb4b067fae5c8a550267a0dcef7708b30d36598b2ed18981711ad9b4a67b23cbf444270f7006d160e50f151ba32fe3402108429d415f7adbfb0be9160fedfda7

data/.gitignore

@@ -0,0 +1,31 @@
+*.bundle
+*.gem
+*.iml
+*.stTheme.cache
+*.sublime-project
+*.sublime-workspace
+*.swp
+*.tmlanguage.cache
+*.tmPreferences.cache
+*~
+.byebug_history
+.DS_Store
+.idea/
+.ruby-gemset
+.ruby-version
+.tags
+.tags1
+bbin/
+binstubs/*
+bundler_stubs/*/.yardoc
+Gemfile.lock
+/.bundle/
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,11 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.1
+  - 2.4
+  - 2.5
+  - 2.6
+  - jruby-9.1.9.0
+before_install:
+  - gem update --system
+  - gem install bundler

data/BENCHMARK.md

@@ -0,0 +1,50 @@
+Results of `rake:benchmark` on ruby 2.6.0preview1 (2018-02-24 trunk 62554) [x86_64-darwin17]
+
+```
+Detecting non-whitespace
+
+ CharacterSet#cover?: 13244577.7 i/s
+       Regexp#match?:  8027017.5 i/s - 1.65x  slower
+```
+```
+Detecting non-letters
+
+ CharacterSet#cover?: 13082940.8 i/s
+       Regexp#match?:  5372589.2 i/s - 2.44x  slower
+```
+```
+Removing whitespace
+
+CharacterSet#delete_in:   389315.6 i/s
+         String#gsub:   223773.5 i/s - 1.74x  slower
+```
+```
+Removing whitespace, emoji and umlauts
+
+CharacterSet#delete_in:   470239.3 i/s
+         String#gsub:   278679.4 i/s - 1.69x  slower
+```
+```
+Removing non-whitespace
+
+CharacterSet#keep_in:  1138461.0 i/s
+         String#gsub:   235287.4 i/s - 4.84x  slower
+```
+```
+Extracting emoji
+
+CharacterSet#keep_in:  1474472.0 i/s
+         String#gsub:   212269.6 i/s - 6.95x  slower
+```
+```
+Detecting whitespace
+
+CharacterSet#used_by?: 13063108.7 i/s
+       Regexp#match?:  7215075.0 i/s - 1.81x  slower
+```
+```
+Detecting emoji in a large string
+
+CharacterSet#used_by?:   246527.7 i/s
+       Regexp#match?:    92956.5 i/s - 2.65x  slower
+```

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in character_set.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Janosch Müller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,180 @@
+# CharacterSet
+
+[![Gem Version](https://badge.fury.io/rb/character_set.svg)](http://badge.fury.io/rb/character_set)
+[![Build Status](https://travis-ci.org/janosch-x/character_set.svg?branch=master)](https://travis-ci.org/janosch-x/character_set)
+
+A gem to build, read, write and compare sets of Unicode codepoints.
+
+Many parts can be used independently, e.g.:
+- `CharacterSet::Character`
+- `CharacterSet::Parser`
+- `CharacterSet::Writer`
+- [`RangeCompressor`](https://github.com/janosch-x/range_compressor)
+
+## Usage
+
+### Parse/Initialize
+
+These all produce a `CharacterSet` containing `a`, `b` and `c`:
+
+```ruby
+CharacterSet['a', 'b', 'c']
+CharacterSet[97, 98, 99]
+CharacterSet.new('a'..'c')
+CharacterSet.new(0x61..0x63)
+CharacterSet.of('abacababa')
+CharacterSet.parse('[a-c]')
+CharacterSet.parse('\U00000061-\U00000063')
+```
+
+If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/janosch-x/regexp_property_values) are installed, `::of_regexp` and `::of_property` can also be used. `::of_regexp` can handle intersections, negations, and set nesting:
+
+```ruby
+# are there any non-digit ascii chars classified as emoji?
+set = CharacterSet.of_regexp(/[\D&&[:ascii:]&&\p{emoji}]/)
+
+# ... of course there are!
+set.to_a(stringify: true) # => ["#", "*"]
+
+# with the core extension:
+require 'character_set/core_ext/regexp_ext'
+/[a-e&&[^c]]/.character_set # => CharacterSet['a', 'b', 'd', 'e']
+```
+
+### Common utility sets
+
+```ruby
+CharacterSet.ascii
+CharacterSet.bmp
+CharacterSet.crypt
+CharacterSet.emoji
+CharacterSet.newline
+CharacterSet.unicode
+CharacterSet.url_fragment
+CharacterSet.url_host
+CharacterSet.url_path
+CharacterSet.url_query
+CharacterSet.whitespace
+
+# e.g.
+CharacterSet.url_query.cover?('?a=(b$c;)') # => true
+CharacterSet.emoji.sample(5) # => ["⛷", "👈", "🌞", "♑", "⛈"]
+
+# all can be prefixed with `non_`, e.g.
+CharacterSet.non_ascii.delete_in(string)
+```
+
+### Interact with Strings
+
+CharacterSet can replace some `Regexp` actions on Strings, at better speed (see [benchmarks](./BENCHMARK.md)).
+
+`#used_by?` and `#cover?` can replace some `Regexp#match?` calls:
+
+```ruby
+CharacterSet.ascii.used_by?('Tüür') # => true
+CharacterSet.ascii.cover?('Tüür') # => false
+CharacterSet.ascii.cover?('Tr') # => true
+```
+
+`#delete_in(!)` and `#keep_in(!)` can replace `String#gsub(!)` and the like:
+```ruby
+string = 'Tüür'
+
+CharacterSet.ascii.delete_in(string) # => 'üü'
+CharacterSet.ascii.keep_in(string) # => 'Tr'
+string # => 'Tüür'
+
+CharacterSet.ascii.delete_in!(string) # => 'üü'
+string # => 'üü'
+CharacterSet.ascii.keep_in!(string) # => ''
+string # => ''
+```
+
+There is also a core extension for String interaction.
+```ruby
+require 'character_set/core_ext/string_ext'
+
+"a\rb".character_set & CharacterSet.newline # => CharacterSet["\r"]
+"a\rb".uses_character_set?(CharacterSet.emoji) # => false
+"a\rb".covered_by_character_set?(CharacterSet.newline) # => false
+"a\rb".delete_character_set(CharacterSet.newline) # => 'ab'
+# etc.
+```
+
+### Manipulate
+
+Use any [Ruby Set method](https://ruby-doc.org/stdlib-2.5.1/libdoc/set/rdoc/Set.html), e.g. `#+`, `#-`, `#&`, `#^`, `#intersect?`, `#<`, `#>` etc. to interact with other sets. Use `#add`, `#delete`, `#include?` etc. to change or check for members.
+
+Where appropriate, methods take both chars and codepoints, e.g.:
+
+```ruby
+CharacterSet['a'].add('b') # => CharacterSet['a', 'b']
+CharacterSet['a'].add(98) # => CharacterSet['a', 'b']
+CharacterSet['a'].include?('a') # => true
+CharacterSet['a'].include?(0x61) # => true
+```
+
+`#inversion` can be used to create a `CharacterSet` with all valid Unicode codepoints that are not in the current set:
+
+```ruby
+non_a = CharacterSet['a'].inversion
+# => #<CharacterSet (size: 1112063)>
+
+non_a.include?('a') # => false
+non_a.include?('ü') # => true
+
+# surrogate pair halves are not included by default
+CharacterSet['a'].inversion(include_surrogates: true)
+# => #<CharacterSet (size: 1114111)>
+```
+
+`#case_insensitive` can be used to create a `CharacterSet` where upper/lower case codepoints are supplemented:
+
+```ruby
+CharacterSet['1', 'a'].case_insensitive # => CharacterSet['1', 'A', 'a']
+```
+
+### Write
+```ruby
+set = CharacterSet['a', 'b', 'c', 'j', '-']
+
+# safely printable ASCII chars are not escaped by default
+set.to_s # => 'a-cj\x2D'
+set.to_s(escape_all: true) # => '\x61-\x63\x6A\x2D'
+
+# brackets may be added
+set.to_s(in_brackets: true) # => '[a-cj\x2D]'
+
+# the default escape format is Ruby/ES6 compatible, others are available
+set = CharacterSet['a', 'b', 'c', 'ɘ', '🤩']
+set.to_s # => 'a-c\u0258\u{1F929}'
+set.to_s(format: 'U+') # => 'a-cU+0258U+1F929'
+set.to_s(format: 'Python') # => "a-c\u0258\U0001F929"
+set.to_s(format: 'raw') # => 'a-cɘ🤩'
+
+# or pass a block
+set.to_s { |char| "[#{char.codepoint}]" } # => "a-c[600][129321]"
+set.to_s(escape_all: true) { |c| "<#{c.hex}>" } # => "<61>-<63><258><1F929>"
+
+# disable abbreviation (grouping of codepoints in ranges)
+set.to_s(abbreviate: false) # => "abc\u0258\u{1F929}"
+
+# for full js regex compatibility in case of astral members:
+set.to_s_with_surrogate_alternation # => '(?:[\u0258]|\ud83e\udd29)'
+```
+
+### Unicode plane methods
+
+There are some methods to check for planes and to handle [BMP](https://en.wikipedia.org/wiki/Plane_%28Unicode%29#Basic_Multilingual_Plane) and astral parts:
+```Ruby
+CharacterSet['a', 'ü', '🤩'].bmp_part # => CharacterSet['a', 'ü']
+CharacterSet['a', 'ü', '🤩'].astral_part # => CharacterSet['🤩']
+CharacterSet['a', 'ü', '🤩'].bmp_ratio # => 0.6666666
+CharacterSet['a', 'ü', '🤩'].planes # => [0, 1]
+CharacterSet['a', 'ü', '🤩'].member_in_plane?(7) # => false
+CharacterSet::Character.new('a').plane # => 0
+```
+
+### Contributions
+
+Feel free to send suggestions, point out issues, or submit pull requests.

data/Rakefile

@@ -0,0 +1,137 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubygems/package_task'
+require 'rake/extensiontask'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec
+
+Rake::ExtensionTask.new('character_set') do |ext|
+  ext.lib_dir = 'lib/character_set'
+end
+
+namespace :java do
+  java_gemspec = eval File.read('./character_set.gemspec')
+  java_gemspec.platform = 'java'
+  java_gemspec.extensions = []
+
+  Gem::PackageTask.new(java_gemspec) do |pkg|
+    pkg.need_zip = true
+    pkg.need_tar = true
+    pkg.package_dir = 'pkg'
+  end
+end
+
+task package: 'java:gem'
+
+desc 'Download relevant ruby/spec tests, adapt to CharacterSet and its variants'
+task :sync_ruby_spec do
+  require 'fileutils'
+
+  variants = {
+    'CharacterSet'       => './spec/ruby-spec/library/character_set',
+    'CharacterSet::Pure' => './spec/ruby-spec/library/character_set_pure',
+  }
+  variants.each do |_, dir|
+    FileUtils.rm_rf(dir) if File.exist?(dir)
+    `svn export https://github.com/ruby/spec/trunk/library/set/sortedset #{dir}`
+  end
+
+  base = variants.first[1]
+  variants.each_value { |dir| FileUtils.copy_entry(base, dir) unless dir == base }
+
+  variants.each.with_index do |(class_name, dir), i|
+    Dir["#{dir}/**/*.rb"].each do |spec|
+      # remove some tests that do not apply or are covered otherwise
+      if spec =~ %r{/(flatten|initialize|pretty_print)}
+        File.delete(spec)
+        next
+      end
+
+      # some examples w. Strings must be adapted, "mspec" made rspec-compatible,
+      # and `i` added to shared example names or they'll override each other
+      adapted_content =
+        File
+        .read(spec)
+        .gsub('SortedSet', class_name)
+        .gsub('sorted_set_', "sorted_set_#{i}_")
+        .gsub(/describe (.*), shared.*$/, 'shared_examples \1 do |method|')
+        .gsub(/1\.0|"cat"|"dog"|"hello"|"test"/, '0')
+        .gsub('"one"', '1')
+        .gsub('"two"', '2')
+        .gsub('"three"', '3')
+        .gsub('"four"', '4')
+        .gsub('"five"', '5')
+        .gsub('@method', 'method')
+        .gsub(/be_(false|true)/, 'be \1')
+        .gsub('mock', 'double')
+
+      File.open(spec, 'w') { |f| f.puts adapted_content }
+    end
+  end
+end
+
+desc 'Download unicode casefold data and write new C header file'
+task :sync_casefold_data do
+  src_path = './CaseFolding.txt'
+  dst_path = './ext/character_set/unicode_casefold_table.h'
+
+  `wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt`
+
+  mapping = File.foreach(src_path).each_with_object({}) do |line, hash|
+    from, type, to = line.split(/\s*;\s*/).first(3)
+    # type 'C' stands for 'common', excludes mappings to multiple chars
+    hash[from] = to if type == 'C'
+  end.sort
+
+  File.open(dst_path, 'w') do |f|
+    f.puts <<-C
+// THIS FILE IS GENERATED BY $ rake sync_casefold_data - DO NOT EDIT'
+
+typedef struct casefold_mapping {
+  unsigned long from;
+  unsigned long to;
+} casefold_mapping;
+
+#define CASEFOLD_COUNT #{mapping.size}
+
+static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
+    C
+
+    mapping.each { |from, to| f.puts "{0x#{from},0x#{to}}," }
+
+    f.puts '};'
+  end
+
+  File.unlink(src_path)
+end
+
+desc 'Run all IPS benchmarks'
+task :benchmark do
+  Dir['./benchmarks/*.rb'].sort.each { |file| require file }
+end
+
+namespace :benchmark do
+  desc 'Run all IPS benchmarks and store the comparison results in BENCHMARK.md'
+  task :write_to_file do
+    $store_comparison_results = {}
+
+    Rake.application[:benchmark].invoke
+
+    File.open('BENCHMARK.md', 'w') do |f|
+      f.puts "Results of `rake:benchmark` on #{RUBY_DESCRIPTION}", ''
+
+      $store_comparison_results.each do |caption, result|
+        f.puts '```', caption, '',
+               result.strip.gsub(/(same-ish).*$/, '\1').lines[1..-1], '```'
+      end
+    end
+  end
+end
+
+unless RUBY_PLATFORM =~ /java/
+  # recompile before benchmarking or running specs
+  task(:benchmark).enhance([:compile])
+  task(:spec).enhance([:compile])
+end

data/benchmarks/cover.rb

@@ -0,0 +1,25 @@
+require_relative './shared'
+
+str = 'Lorem ipsum et dolorem'
+rx = /\S/
+cs = CharacterSet.whitespace.inversion
+
+benchmark(
+  caption: 'Detecting non-whitespace',
+  cases: {
+    'Regexp#match?'       => -> { rx.match?(str) },
+    'CharacterSet#cover?' => -> { cs.cover?(str) },
+  }
+)
+
+str = 'Lorem ipsum et dolorem'
+rx = /[^a-z]/i
+cs = CharacterSet.new('A'..'Z') + CharacterSet.new('a'..'z')
+
+benchmark(
+  caption: 'Detecting non-letters',
+  cases: {
+    'Regexp#match?'       => -> { rx.match?(str) },
+    'CharacterSet#cover?' => -> { cs.cover?(str) },
+  }
+)

data/benchmarks/delete_in.rb

@@ -0,0 +1,25 @@
+require_relative './shared'
+
+str = 'Lorem     ipsum       et      dolorem'
+rx = /\s/
+cs = CharacterSet.whitespace
+
+benchmark(
+  caption: 'Removing whitespace',
+  cases: {
+    'String#gsub'            => -> { str.gsub(rx, '') },
+    'CharacterSet#delete_in' => -> { cs.delete_in(str) },
+  }
+)
+
+str = 'Lörem ipsüm ⛷ et dölörem'
+rx = /[\s\p{emoji}äüö]/
+cs = CharacterSet.whitespace + CharacterSet.emoji + CS['ä', 'ü', 'ö']
+
+benchmark(
+  caption: 'Removing whitespace, emoji and umlauts',
+  cases: {
+    'String#gsub'            => -> { str.gsub(rx, '') },
+    'CharacterSet#delete_in' => -> { cs.delete_in(str) },
+  }
+)

data/benchmarks/keep_in.rb

@@ -0,0 +1,25 @@
+require_relative './shared'
+
+str = 'Lorem ipsum et dolorem'
+rx = /\S/
+cs = CharacterSet.whitespace
+
+benchmark(
+  caption: 'Removing non-whitespace',
+  cases: {
+    'String#gsub'          => -> { str.gsub(rx, '') },
+    'CharacterSet#keep_in' => -> { cs.keep_in(str) },
+  }
+)
+
+str = 'Lorem ipsum ⛷ et dolorem'
+rx = /\p{^emoji}/
+cs = CharacterSet.emoji
+
+benchmark(
+  caption: 'Extracting emoji',
+  cases: {
+    'String#gsub'          => -> { str.gsub(rx, '') },
+    'CharacterSet#keep_in' => -> { cs.keep_in(str) },
+  }
+)

data/benchmarks/shared.rb

@@ -0,0 +1,25 @@
+lib = File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'benchmark/ips'
+require 'character_set'
+
+def benchmark(caption: nil, cases: {})
+  puts caption
+
+  report = Benchmark.ips do |x|
+    cases.each do |label, callable|
+      x.report(label, &callable)
+    end
+    x.compare!
+  end
+
+  return unless $store_comparison_results
+
+  old_stdout = $stdout.clone
+  captured_stdout = StringIO.new
+  $stdout = captured_stdout
+  report.run_comparison
+  $store_comparison_results[caption] = captured_stdout.string
+  $stdout = old_stdout
+end

data/benchmarks/used_by.rb

@@ -0,0 +1,25 @@
+require_relative './shared'
+
+str = 'Lorem ipsum et dolorem'
+rx = /\s/
+cs = CharacterSet.whitespace
+
+benchmark(
+  caption: 'Detecting whitespace',
+  cases: {
+    'Regexp#match?'         => -> { rx.match?(str) },
+    'CharacterSet#used_by?' => -> { cs.used_by?(str) },
+  }
+)
+
+str = 'Lorem ipsum et dolorem' * 20 + '⛷' + 'Lorem ipsum et dolorem' * 20
+rx = /\p{emoji}/
+cs = CharacterSet.emoji
+
+benchmark(
+  caption: 'Detecting emoji in a large string',
+  cases: {
+    'Regexp#match?'         => -> { rx.match?(str) },
+    'CharacterSet#used_by?' => -> { cs.used_by?(str) },
+  }
+)

data/bin/console

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+
+require 'character_set'
+require 'character_set/core_ext'
+require 'character_set/pure'
+
+require 'regexp_property_values'
+
+CS = CharacterSet
+CP = CharacterSet::Pure
+PV = RegexpPropertyValues
+
+require 'benchmark'
+def m(&block); Benchmark.measure(&block); end
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/character_set.gemspec

@@ -0,0 +1,34 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'character_set/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'character_set'
+  s.version       = CharacterSet::VERSION
+  s.authors       = ['Janosch Müller']
+  s.email         = ['janosch84@gmail.com']
+
+  s.summary       = 'Build, read, write and compare sets of Unicode codepoints.'
+  s.homepage      = 'https://github.com/janosch-x/character_set'
+  s.license       = 'MIT'
+
+  s.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  s.require_paths = ['lib']
+
+  s.extensions  = %w[ext/character_set/extconf.rb]
+
+  s.required_ruby_version = '>= 2.1.0'
+
+  s.add_dependency 'range_compressor', '~> 1.0'
+
+  s.add_development_dependency 'benchmark-ips', '~> 2.7'
+  s.add_development_dependency 'bundler', '~> 1.16'
+  s.add_development_dependency 'rake', '~> 10.0'
+  s.add_development_dependency 'rake-compiler', '~> 1.0'
+  s.add_development_dependency 'regexp_parser', '~> 1.0'
+  s.add_development_dependency 'regexp_property_values', '~> 0.3.2'
+  s.add_development_dependency 'rspec', '~> 3.0'
+end