character_set 1.2.0 → 1.5.0

data/lib/character_set/ruby_fallback.rb

@@ -1,12 +1,14 @@
-require 'set'
+if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+  require 'sorted_set'
+else
+  require 'set'
+end
 require 'character_set/ruby_fallback/set_methods'
-require 'character_set/ruby_fallback/plane_methods'
 require 'character_set/ruby_fallback/character_set_methods'
 
 class CharacterSet
   module RubyFallback
     include CharacterSet::RubyFallback::SetMethods
-    include CharacterSet::RubyFallback::PlaneMethods
     include CharacterSet::RubyFallback::CharacterSetMethods
 
     def self.prepended(klass)

data/lib/character_set/set_method_adapters.rb

@@ -22,13 +22,14 @@ class CharacterSet
 
     # Allow some methods to take an Enum just as well as another CharacterSet.
     # Tested by ruby-spec.
-    %w[& + - ^ | difference intersection subtract union].each do |method|
+    %w[& + - ^ | difference disjoint? intersect? intersection
+       subtract union].each do |method|
       class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{method}(arg)
           if arg.is_a?(CharacterSet)
-            super
+            super(arg)
           elsif arg.respond_to?(:each)
-            super(CharacterSet.new(arg.to_a))
+            super(self.class.new(arg.to_a))
           else
             raise ArgumentError, 'pass an enumerable'
           end

data/lib/character_set/shared_methods.rb

@@ -22,33 +22,29 @@ class CharacterSet
           end
 
           def of_property(property_name)
-            require_optional_dependency('regexp_property_values')
+            require_optional_dependency('regexp_property_values', __method__)
 
             property = RegexpPropertyValues[property_name.to_s]
             from_ranges(*property.matched_ranges)
           end
 
           def of_regexp(regexp)
-            require_optional_dependency('regexp_parser')
+            require_optional_dependency('regexp_parser', __method__)
 
             root = ::Regexp::Parser.parse(regexp)
             of_expression(root)
           end
 
           def of_expression(expression)
-            ExpressionConverter.convert(expression)
+            ExpressionConverter.convert(expression, self)
           end
 
-          def require_optional_dependency(name)
+          def require_optional_dependency(name, method)
             required_optional_dependencies[name] ||= begin
               require name
               true
             rescue ::LoadError
-              entry_point = caller_locations.reverse.find do |loc|
-                loc.absolute_path.to_s.include?('/lib/character_set')
-              end
-              method = entry_point && entry_point.label
-              raise LoadError, 'You must the install the optional dependency '\
+              raise LoadError, 'You must install the optional dependency '\
                                "'\#{name}' to use the method `\#{method}'."
             end
           end
@@ -70,37 +66,64 @@ class CharacterSet
           merge(enum)
         end
 
-        # stringification methods
+        # CharacterSet-specific conversion methods
+
+        def assigned_part
+          self & self.class.assigned
+        end
+
+        def valid_part
+          self - self.class.surrogate
+        end
+
+        # CharacterSet-specific stringification methods
 
         def to_s(opts = {}, &block)
           Writer.write(ranges, opts, &block)
         end
 
+        def to_s_with_surrogate_ranges
+          Writer.write_surrogate_ranges(bmp_part.ranges, astral_part.ranges)
+        end
+
         def to_s_with_surrogate_alternation
           Writer.write_surrogate_alternation(bmp_part.ranges, astral_part.ranges)
         end
 
+        def secure_token(length = 32)
+          CharacterSet.require_optional_dependency('securerandom', __method__)
+          cps = to_a
+          len = cps.count
+          1.upto(length).map { cps[SecureRandom.random_number(len)] }.pack('U*')
+        end
+        alias random_token secure_token
+
         def inspect
           len = length
-          "#<CharacterSet: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"
+          "#<#{klass.name}: {\#{first(5) * ', '}\#{'...' if len > 5}} (size: \#{len})>"
         end
 
-        # unicode-plane-related methods
+        # C-extension adapter methods. Need overriding in pure fallback.
+        # Parsing kwargs in C is slower, verbose, and kinda deprecated.
 
-        def bmp_part?
-          !bmp_part.empty?
+        def inversion(include_surrogates: false, upto: 0x10FFFF)
+          ext_inversion(include_surrogates, upto)
         end
 
-        def astral_part?
-          !astral_part.empty?
+        def section(from:, upto: 0x10FFFF)
+          ext_section(from, upto)
         end
 
-        def bmp_ratio
-          bmp_part.count / count.to_f
+        def count_in_section(from:, upto: 0x10FFFF)
+          ext_count_in_section(from, upto)
         end
 
-        def astral_ratio
-          astral_part.count / count.to_f
+        def section?(from:, upto: 0x10FFFF)
+          ext_section?(from, upto)
+        end
+
+        def section_ratio(from:, upto: 0x10FFFF)
+          ext_section_ratio(from, upto)
         end
 
         #
@@ -136,42 +159,38 @@ class CharacterSet
         end
 
         def divide(&func)
-          block_given? or return enum_for(__method__) { size }
-          require 'set'
-
-          if func.arity == 2
-            require 'tsort'
+          CharacterSet.require_optional_dependency('set', __method__)
+          Set.new(to_a).divide(&func)
+        end
+      RUBY
 
-            class << dig = {}
-              include TSort
+      # CharacterSet-specific section methods
 
-              alias tsort_each_node each_key
-              def tsort_each_child(node, &block)
-                fetch(node).each(&block)
-              end
-            end
+      {
+        ascii:  0..0x7F,
+        bmp:    0..0xFFFF,
+        astral: 0x10000..0x10FFFF,
+      }.each do |section_name, range|
+        klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{section_name}_part
+            section(from: #{range.begin}, upto: #{range.end})
+          end
 
-            each do |u|
-              dig[u] = a = []
-              each{ |v| a << v if yield(u, v) }
-            end
+          def #{section_name}_part?
+            section?(from: #{range.begin}, upto: #{range.end})
+          end
 
-            set = Set.new
-            dig.each_strongly_connected_component do |css|
-              set.add(self.class.new(css))
-            end
-            set
-          else
-            Set.new(classify(&func).values)
+          def #{section_name}_only?
+            #{range.begin == 0 ?
+              "!section?(from: #{range.end}, upto: 0x10FFFF)" :
+              "!section?(from: 0, upto: #{range.begin})"}
           end
-        end
 
-        # C-extension adapter method. Needs overriding in pure fallback.
-        # Parsing kwargs in C is slower, verbose, and kinda deprecated.
-        def inversion(include_surrogates: false, upto: 0x10FFFF)
-          ext_inversion(include_surrogates, upto)
-        end
-      RUBY
+          def #{section_name}_ratio
+            section_ratio(from: #{range.begin}, upto: #{range.end})
+          end
+        RUBY
+      end
     end # self.included
   end # SharedMethods
 end

data/lib/character_set/version.rb

@@ -1,3 +1,3 @@
 class CharacterSet
-  VERSION = '1.2.0'
+  VERSION = '1.5.0'
 end

data/lib/character_set/writer.rb

@@ -1,37 +1,108 @@
 class CharacterSet
   module Writer
-    module_function
-
-    def write(codepoint_ranges, opts = {}, &block)
-      content = codepoint_ranges.map do |range|
-        if range.size > 2 && opts[:abbreviate] != false
-          range.minmax.map { |cp| Character.new(cp).escape(opts, &block) }.join('-')
-        else
-          range.map { |cp| Character.new(cp).escape(opts, &block) }.join
+    class << self
+      def write(codepoint_ranges, opts = {}, &block)
+        content = codepoint_ranges.map do |range|
+          if range.size > 2 && opts[:abbreviate] != false
+            bounds = [range.min, range.max]
+            bounds.map { |cp| write_codepoint(cp, opts, &block) }.join('-')
+          else
+            range.map { |cp| write_codepoint(cp, opts, &block) }.join
+          end
+        end.join
+        opts[:in_brackets] ? "[#{content}]" : content
+      end
+
+      def write_codepoint(codepoint, opts = {}, &block)
+        Character.new(codepoint).escape(opts, &block)
+      end
+
+      def write_surrogate_ranges(bmp_ranges, astral_ranges)
+        astral_branches = surrogate_range_expressions(astral_ranges)
+        bmp_set_with_alternatives(bmp_ranges, astral_branches)
+      end
+
+      def write_surrogate_alternation(bmp_ranges, astral_ranges)
+        astral_branches = surrogate_pairs(astral_ranges)
+        bmp_set_with_alternatives(bmp_ranges, astral_branches)
+      end
+
+      private
+
+      def surrogate_range_expressions(astral_ranges)
+        compressed_surrogate_range_pairs(astral_ranges).map do |hi_ranges, lo_ranges|
+          [hi_ranges, lo_ranges].map do |ranges|
+            use_brackets = ranges.size > 1 || ranges.first.size > 1
+            write(ranges, format: :js, in_brackets: use_brackets)
+          end.join
         end
-      end.join
-      opts[:in_brackets] ? "[#{content}]" : content
-    end
+      end
+
+      def compressed_surrogate_range_pairs(astral_ranges)
+        halves = astral_ranges.flat_map { |range| surrogate_half_ranges(range) }
 
-    def write_surrogate_alternation(bmp_ranges, astral_ranges)
-      bmp_set = write(bmp_ranges, format: :js, in_brackets: true)
-      if astral_ranges.empty?
-        bmp_set
-      else
-        surrogate_pairs = surrogate_pairs(astral_ranges)
-        "(?:#{((bmp_ranges.any? ? [bmp_set] : []) + surrogate_pairs) * '|'})"
+        # compress high surrogate codepoint ranges with common low range half
+        with_common_lo = halves.group_by(&:last).map do |lo_range, pairs|
+          hi_ranges = pairs.map(&:first)
+          compressed_hi_ranges = hi_ranges.each_with_object([]) do |range, arr|
+            prev = arr.last
+            if prev.nil? || prev.max + 1 < range.min # first or gap
+              arr << range
+            else # continuous codepoints, expand previous range
+              arr[-1] = (prev.min)..(range.max)
+            end
+          end
+          [compressed_hi_ranges, lo_range]
+        end
+
+        # compress low surrogate codepoint ranges with common high ranges
+        with_common_lo.each_with_object({}) do |(hi_ranges, lo_range), hash|
+          (hash[hi_ranges] ||= []) << lo_range
+        end
       end
-    end
 
-    def surrogate_pairs(astral_ranges)
-      astral_ranges.flat_map { |range| range.map { |cp| surrogate_pair(cp) } }
-    end
+      def surrogate_half_ranges(astral_range)
+        hi_min, lo_min = surrogate_pair_codepoints(astral_range.min)
+        hi_max, lo_max = surrogate_pair_codepoints(astral_range.max)
+        hi_count = 1 + hi_max - hi_min
+        return [[hi_min..hi_min, lo_min..lo_max]] if hi_count == 1
+
+        ranges = []
+
+        # first high surrogate might be partially covered (if lo_min > 0xDC00)
+        ranges << [hi_min..hi_min, lo_min..0xDFFF]
+
+        # any high surrogates in between are fully covered
+        ranges << [(hi_min + 1)..(hi_max - 1), 0xDC00..0xDFFF] if hi_count > 2
 
-    def surrogate_pair(astral_codepoint)
-      base = astral_codepoint - 0x10000
-      high = ((base / 1024).floor + 0xD800).to_s(16)
-      low  = (base % 1024 + 0xDC00).to_s(16)
-      "\\u#{high}\\u#{low}"
+        # last high surrogate might be partially covered (if lo_max < 0xDFFF)
+        ranges << [hi_max..hi_max, 0xDC00..lo_max]
+
+        ranges
+      end
+
+      def surrogate_pair_codepoints(astral_codepoint)
+        base = astral_codepoint - 0x10000
+        high = base / 1024 + 0xD800
+        low  = base % 1024 + 0xDC00
+        [high, low]
+      end
+
+      def bmp_set_with_alternatives(bmp_ranges, alternatives)
+        bmp_set = write(bmp_ranges, format: :js, in_brackets: true)
+        return bmp_set if alternatives.empty? && bmp_ranges.any?
+
+        "(?:#{((bmp_ranges.any? ? [bmp_set] : []) + alternatives).join('|')})"
+      end
+
+      def surrogate_pairs(astral_ranges)
+        astral_ranges.flat_map { |range| range.map { |cp| surrogate_pair(cp) } }
+      end
+
+      def surrogate_pair(astral_codepoint)
+        surrogate_pair_codepoints(astral_codepoint)
+          .map { |half| write_codepoint(half, format: :js) }.join
+      end
     end
   end
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: character_set
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Janosch Müller
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-02 00:00:00.000000000 Z
+date: 2021-12-05 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: sorted_set
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
@@ -24,34 +38,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: range_compressor
   requirement: !ruby/object:Gem::Requirement
@@ -72,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: regexp_property_values
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -108,7 +136,49 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
-description: 
+- !ruby/object:Gem::Dependency
+  name: codecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.12
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.12
+- !ruby/object:Gem::Dependency
+  name: gouteur
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description:
 email:
 - janosch84@gmail.com
 executables: []
@@ -116,26 +186,38 @@ extensions:
 - ext/character_set/extconf.rb
 extra_rdoc_files: []
 files:
+- ".gitattributes"
+- ".github/workflows/gouteur.yml"
+- ".github/workflows/lint.yml"
+- ".github/workflows/tests.yml"
 - ".gitignore"
+- ".gouteur.yml"
 - ".rspec"
-- ".travis.yml"
+- ".rubocop.yml"
 - BENCHMARK.md
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmarks/count_in.rb
 - benchmarks/cover.rb
 - benchmarks/delete_in.rb
 - benchmarks/keep_in.rb
+- benchmarks/scan.rb
 - benchmarks/shared.rb
 - benchmarks/used_by.rb
+- benchmarks/z_add.rb
+- benchmarks/z_delete.rb
+- benchmarks/z_merge.rb
+- benchmarks/z_minmax.rb
 - bin/console
 - bin/setup
 - character_set.gemspec
 - ext/character_set/character_set.c
 - ext/character_set/extconf.rb
 - ext/character_set/unicode_casefold_table.h
+- ext/character_set/unicode_casefold_table.h.tmpl
 - lib/character_set.rb
 - lib/character_set/character.rb
 - lib/character_set/core_ext.rb
@@ -144,10 +226,25 @@ files:
 - lib/character_set/expression_converter.rb
 - lib/character_set/parser.rb
 - lib/character_set/predefined_sets.rb
+- lib/character_set/predefined_sets/any.cps
+- lib/character_set/predefined_sets/ascii.cps
+- lib/character_set/predefined_sets/ascii_alnum.cps
+- lib/character_set/predefined_sets/ascii_letter.cps
+- lib/character_set/predefined_sets/assigned.cps
+- lib/character_set/predefined_sets/bmp.cps
+- lib/character_set/predefined_sets/crypt.cps
+- lib/character_set/predefined_sets/emoji.cps
+- lib/character_set/predefined_sets/newline.cps
+- lib/character_set/predefined_sets/surrogate.cps
+- lib/character_set/predefined_sets/unicode.cps
+- lib/character_set/predefined_sets/url_fragment.cps
+- lib/character_set/predefined_sets/url_host.cps
+- lib/character_set/predefined_sets/url_path.cps
+- lib/character_set/predefined_sets/url_query.cps
+- lib/character_set/predefined_sets/whitespace.cps
 - lib/character_set/pure.rb
 - lib/character_set/ruby_fallback.rb
 - lib/character_set/ruby_fallback/character_set_methods.rb
-- lib/character_set/ruby_fallback/plane_methods.rb
 - lib/character_set/ruby_fallback/set_methods.rb
 - lib/character_set/set_method_adapters.rb
 - lib/character_set/shared_methods.rb
@@ -157,7 +254,7 @@ homepage: https://github.com/jaynetics/character_set
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -172,8 +269,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.0.dev
+signing_key:
 specification_version: 4
 summary: Build, read, write and compare sets of Unicode codepoints.
 test_files: []

data/.travis.yml

@@ -1,8 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.1
-  - 2.4
-  - 2.5
-  - 2.6
-  - jruby-9.1.9.0

data/lib/character_set/ruby_fallback/plane_methods.rb

@@ -1,27 +0,0 @@
-class CharacterSet
-  module RubyFallback
-    module PlaneMethods
-      def bmp_part
-        dup.keep_if { |cp| cp < 0x10000 }
-      end
-
-      def astral_part
-        dup.keep_if { |cp| cp >= 0x10000 }
-      end
-
-      def planes
-        plane_set = {}
-        plane_size = 0x10000.to_f
-        each do |cp|
-          plane = (cp / plane_size).floor
-          plane_set[plane] = true
-        end
-        plane_set.keys
-      end
-
-      def member_in_plane?(num)
-        ((num * 0x10000)...((num + 1) * 0x10000)).any? { |cp| include?(cp) }
-      end
-    end
-  end
-end