net-imap 0.3.7 → 0.4.5

data/net-imap.gemspec

@@ -2,7 +2,7 @@
 
 name = File.basename(__FILE__, ".gemspec")
 version = ["lib", Array.new(name.count("-"), "..").join("/")].find do |dir|
-  break File.foreach(File.join(__dir__, dir, "#{name.tr('-', '/')}.rb")) do |line|
+  break File.foreach(File.join(__dir__, dir, "#{name.tr('-', '/')}.rb"), :encoding=> 'utf-8') do |line|
     /^\s*VERSION\s*=\s*"(.*)"/ =~ line and break $1
   end rescue nil
 end
@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Ruby client api for Internet Message Access Protocol}
   spec.description   = %q{Ruby client api for Internet Message Access Protocol}
   spec.homepage      = "https://github.com/ruby/net-imap"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.3")
   spec.licenses       = ["Ruby", "BSD-2-Clause"]
 
   spec.metadata["homepage_uri"] = spec.homepage
@@ -25,7 +25,8 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z 2>/dev/null`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features|rfcs)/}) }
+    `git ls-files -z 2>/dev/null`.split("\x0")
+      .reject {|f| f.match(%r{^(bin|test|spec|benchmarks|features|rfcs)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

data/rakelib/benchmarks.rake

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+PARSER_TEST_FIXTURES = FileList.new "test/net/imap/fixtures/response_parser/*.yml"
+CLOBBER.include "benchmarks/parser.yml"
+CLEAN.include   "benchmarks/Gemfile-*"
+
+BENCHMARK_INIT = <<RUBY
+  require "yaml"
+  require "net/imap"
+
+  def load_response(file, name)
+    YAML.unsafe_load_file(file).dig(:tests, name, :response)
+      .force_encoding "ASCII-8BIT" \\
+      or abort "ERRORO: missing %p fixture data in %p" % [name, file]
+  end
+
+  parser   = Net::IMAP::ResponseParser.new
+RUBY
+
+file "benchmarks/parser.yml" => PARSER_TEST_FIXTURES do |t|
+  require "yaml"
+  require "pathname"
+  require "net/imap"
+
+  path = Pathname.new(__dir__) / "../test/net/imap/fixtures/response_parser"
+  files = path.glob("*.yml")
+  tests = files.flat_map {|file|
+    file.read
+      .gsub(%r{([-:]) !ruby/(object|struct):\S+}) { $1 }
+      .then {
+        YAML.safe_load(_1, filename: file,
+                       permitted_classes: [Symbol, Regexp], aliases: true)
+      }
+      .fetch(:tests)
+      .select {|test_name, test|
+        :parser_assert_equal == test.fetch(:test_type) {
+          test.key?(:expected) ? :parser_assert_equal : :parser_pending
+        }
+      }
+      .map {|test_name, test| [test_name.to_s, test.fetch(:response)] }
+  }
+
+  benchmarks = tests.map {|fixture_name, response|
+    {"name"    => fixture_name.delete_prefix("test_"),
+     "prelude" => "response = %s" % [response.dump],
+     "script"  => "parser.parse(response)"}
+  }
+    .sort_by { _1["name"] }
+
+  YAML.dump({"prelude" => BENCHMARK_INIT, "benchmark" => benchmarks})
+    .then { File.write t.name, _1 }
+end
+
+namespace :benchmarks do
+  desc "Generate benchmarks from fixture data"
+  task :generate => "benchmarks/parser.yml"
+
+  desc "run the parser benchmarks comparing multiple gem versions"
+  task :compare => :generate do |task, args|
+    cd Pathname.new(__dir__) + ".."
+    current = `git describe --tags --dirty`.chomp
+    current = "dev" if current.empty?
+    versions = args.to_a
+    if versions.empty?
+      latest = %x{git describe --tags --abbrev=0 --match 'v*.*.*'}.chomp
+      versions = latest.empty? ? [] : [latest.delete_prefix("v")]
+    end
+    versions = versions.to_h { [_1, "Gemfile-v#{_1}"] }
+    cd "benchmarks" do
+      versions.each do |version, gemfile|
+        File.write gemfile, <<~RUBY
+          # frozen_string_literal: true
+          source "https://rubygems.org"
+          gem "net-imap", #{version.dump}
+        RUBY
+      end
+      versions = {current => "../Gemfile" , **versions}.map {
+        "%s::/usr/bin/env BUNDLE_GEMFILE=%s ruby" % _1
+      }.join(";")
+
+      extra = ENV.fetch("BENCHMARK_ARGS", "").shellsplit
+
+      sh("benchmark-driver",
+         "--bundler",
+         "-e", versions,
+         "parser.yml",
+         *extra)
+    end
+  end
+
+end

data/rakelib/saslprep.rake

@@ -10,17 +10,17 @@ end
 
 directory "lib/net/imap/sasl"
 
-file "lib/net/imap/sasl/stringprep_tables.rb" => generator.rb_deps do |t|
+file "lib/net/imap/stringprep/tables.rb" => generator.rb_deps do |t|
   File.write t.name, generator.stringprep_rb
 end
 
-file "lib/net/imap/sasl/saslprep_tables.rb" => generator.rb_deps do |t|
+file "lib/net/imap/stringprep/saslprep_tables.rb" => generator.rb_deps do |t|
   File.write t.name, generator.saslprep_rb
 end
 
 GENERATED_RUBY = FileList.new(
-  "lib/net/imap/sasl/stringprep_tables.rb",
-  "lib/net/imap/sasl/saslprep_tables.rb",
+  "lib/net/imap/stringprep/tables.rb",
+  "lib/net/imap/stringprep/saslprep_tables.rb",
 )
 
 CLEAN.include   generator.clean_deps

data/rakelib/string_prep_tables_generator.rb

@@ -62,9 +62,9 @@ class StringPrepTablesGenerator
       # This file is generated from RFC3454, by rake.  Don't edit directly.
       #++
 
-      module Net::IMAP::SASL
+      module Net::IMAP::StringPrep
 
-        module StringPrep
+        module Tables
 
           #{asgn_table "A.1"}
 
@@ -74,6 +74,12 @@ class StringPrepTablesGenerator
 
           #{asgn_table "B.3"}
 
+          #{asgn_mapping "B.1", ""}
+
+          #{asgn_mapping "B.2"}
+
+          #{asgn_mapping "B.3"}
+
           #{asgn_table "C.1.1"}
 
           #{asgn_table "C.1.2"}
@@ -105,14 +111,16 @@ class StringPrepTablesGenerator
 
           BIDI_DESC_REQ2 = "A string with RandALCat characters must not contain LCat characters."
 
-          # Bidirectional Characters [StringPrep, §6], Requirement 2::
+          # Bidirectional Characters [StringPrep, §6], Requirement 2
+          # >>>
           #   If a string contains any RandALCat character, the string MUST NOT
           #   contain any LCat character.
           BIDI_FAILS_REQ2 = #{bidi_fails_req2.inspect}.freeze
 
           BIDI_DESC_REQ3 = "A string with RandALCat characters must start and end with RandALCat characters."
 
-          # Bidirectional Characters [StringPrep, §6], Requirement 3::
+          # Bidirectional Characters [StringPrep, §6], Requirement 3
+          # >>>
           #   If a string contains any RandALCat character, a RandALCat
           #   character MUST be the first character of the string, and a
           #   RandALCat character MUST be the last character of the string.
@@ -122,15 +130,21 @@ class StringPrepTablesGenerator
           BIDI_FAILURE = #{bidi_failure_regexp.inspect}.freeze
 
           # Names of each codepoint table in the RFC-3454 appendices
-          TABLE_TITLES = {
+          TITLES = {
             #{table_titles_rb}
           }.freeze
 
           # Regexps matching each codepoint table in the RFC-3454 appendices
-          TABLE_REGEXPS = {
+          REGEXPS = {
             #{table_regexps_rb}
           }.freeze
 
+          MAPPINGS = {
+            "B.1" => [IN_B_1, MAP_B_1].freeze,
+            "B.2" => [IN_B_2, MAP_B_2].freeze,
+            "B.3" => [IN_B_3, MAP_B_3].freeze,
+          }.freeze
+
         end
       end
     RUBY
@@ -157,27 +171,30 @@ class StringPrepTablesGenerator
       # This file is generated from RFC3454, by rake.  Don't edit directly.
       #++
 
-      module Net::IMAP::SASL
+      module Net::IMAP::StringPrep
 
         module SASLprep
 
           # RFC4013 §2.1 Mapping - mapped to space
-          # * non-ASCII space characters (\\StringPrep\\[\\"C.1.2\\"]) that can be
-          #   mapped to SPACE (U+0020), and
+          # >>>
+          #   non-ASCII space characters (\\StringPrep\\[\\"C.1.2\\"]) that can
+          #   be mapped to SPACE (U+0020)
           #
           # Equal to \\StringPrep\\[\\"C.1.2\\"].
-          # Redefined here to avoid loading the StringPrep module.
+          # Redefined here to avoid loading StringPrep::Tables unless necessary.
           MAP_TO_SPACE = #{regex_str "C.1.2"}
 
           # RFC4013 §2.1 Mapping - mapped to nothing
-          #   the "commonly mapped to nothing" characters (\\StringPrep\\[\\"B.1\\"])
-          #   that can be mapped to nothing.
+          # >>>
+          #   the "commonly mapped to nothing" characters
+          #   (\\StringPrep\\[\\"B.1\\"]) that can be mapped to nothing.
           #
           # Equal to \\StringPrep\\[\\"B.1\\"].
-          # Redefined here to avoid loading the StringPrep module.
+          # Redefined here to avoid loading StringPrep::Tables unless necessary.
           MAP_TO_NOTHING = #{regex_str "B.1"}
 
-          # RFC4013 §2.3 Prohibited Output::
+          # RFC4013 §2.3 Prohibited Output
+          # >>>
           # * Non-ASCII space characters — \\StringPrep\\[\\"C.1.2\\"]
           # * ASCII control characters — \\StringPrep\\[\\"C.2.1\\"]
           # * Non-ASCII control characters — \\StringPrep\\[\\"C.2.2\\"]
@@ -192,45 +209,52 @@ class StringPrepTablesGenerator
 
           # Adds unassigned (by Unicode 3.2) codepoints to TABLES_PROHIBITED.
           #
-          # RFC4013 §2.5 Unassigned Code Points::
-          #   This profile specifies the \\StringPrep\\[\\"A.1\\"] table as its list of
-          #   unassigned code points.
+          # RFC4013 §2.5 Unassigned Code Points
+          # >>>
+          #   This profile specifies the \\StringPrep\\[\\"A.1\\"] table as its
+          #   list of unassigned code points.
           TABLES_PROHIBITED_STORED = ["A.1", *TABLES_PROHIBITED].freeze
 
-          # Matches codepoints prohibited by RFC4013 §2.3.
+          # A Regexp matching codepoints prohibited by RFC4013 §2.3.
           #
-          # See TABLES_PROHIBITED.
-          #
-          # Equal to +Regexp.union+ of the TABLES_PROHIBITED tables.  Redefined
-          # here to avoid loading the StringPrep module unless necessary.
+          # This combines all of the TABLES_PROHIBITED tables.
           PROHIBITED_OUTPUT = #{regex_str(*SASL_TABLES_PROHIBITED)}
 
-          # RFC4013 §2.5 Unassigned Code Points::
-          #   This profile specifies the \\StringPrep\\[\\"A.1\\"] table as its list of
-          #   unassigned code points.
+          # RFC4013 §2.5 Unassigned Code Points
+          # >>>
+          #   This profile specifies the \\StringPrep\\[\\"A.1\\"] table as its
+          #   list of unassigned code points.
+          #
+          # Equal to \\StringPrep\\[\\"A.1\\"].
+          # Redefined here to avoid loading StringPrep::Tables unless necessary.
           UNASSIGNED = #{regex_str "A.1"}
 
-          # Matches codepoints prohibited by RFC4013 §2.3 and §2.5.
+          # A Regexp matching codepoints prohibited by RFC4013 §2.3 and §2.5.
           #
-          # See TABLES_PROHIBITED_STORED.
+          # This combines PROHIBITED_OUTPUT and UNASSIGNED.
           PROHIBITED_OUTPUT_STORED = Regexp.union(
             UNASSIGNED, PROHIBITED_OUTPUT
           ).freeze
 
           # Bidirectional Characters [StringPrep, §6]
+          #
+          # A Regexp for strings that don't satisfy StringPrep's Bidirectional
+          # Characters rules.
+          #
+          # Equal to StringPrep::Tables::BIDI_FAILURE.
+          # Redefined here to avoid loading StringPrep::Tables unless necessary.
           BIDI_FAILURE = #{bidi_failure_regexp.inspect}.freeze
 
-          # Matches strings prohibited by RFC4013 §2.3 and §2.4.
+          # A Regexp matching strings prohibited by RFC4013 §2.3 and §2.4.
           #
-          # This checks prohibited output and bidirectional characters.
+          # This combines PROHIBITED_OUTPUT and BIDI_FAILURE.
           PROHIBITED = Regexp.union(
             PROHIBITED_OUTPUT, BIDI_FAILURE,
           )
 
-          # Matches strings prohibited by RFC4013 §2.3, §2.4, and §2.5.
+          # A Regexp matching strings prohibited by RFC4013 §2.3, §2.4, and §2.5.
           #
-          # This checks prohibited output, bidirectional characters, and
-          # unassigned codepoints.
+          # This combines PROHIBITED_OUTPUT_STORED and BIDI_FAILURE.
           PROHIBITED_STORED = Regexp.union(
             PROHIBITED_OUTPUT_STORED, BIDI_FAILURE,
           )
@@ -284,6 +308,15 @@ class StringPrepTablesGenerator
       .map{|s,e| s..(e || s)}
   end
 
+  # TODO: DRY with unicode_normalize
+  def to_map(table)
+    table = table.to_hash
+      .transform_keys { Integer _1, 16 }
+      .transform_keys { [_1].pack("U*") }
+      .transform_values {|cps| cps.map { Integer _1, 16 } }
+      .transform_values { _1.pack("U*") }
+  end
+
   # Starting from a codepoints array (rather than ranges) to deduplicate merged
   # tables.
   def to_regexp(codepoints, negate: false)
@@ -352,6 +385,13 @@ class StringPrepTablesGenerator
     asgn_regex(name, regexp_for(name, negate: negate), negate: negate)
   end
 
+  def asgn_mapping(name, replacement = to_map(tables[name]))
+    cname = name.tr(?., ?_).upcase
+    "# Replacements for %s\n%s%s = %p.freeze" % [
+      "IN_#{name}", "  " * 2, "MAP_#{cname}", replacement,
+    ]
+  end
+
   def regexp_const_desc(name, negate: false)
     if negate then "Matches the negation of the %s table" % [name]
     else %q{%s \\StringPrep\\[\\"%s\\"]} % [titles.fetch(name), name]
@@ -376,40 +416,22 @@ class StringPrepTablesGenerator
   def bidi_L        ; regexp_for "D.2" end
 
   def bidi_fails_req2
-    / # RandALCat followed by LCat
-      (?<r_and_al_cat>#{bidi_R_AL.source})
-        .*?
-        (?<l_cat>#{bidi_L.source})
-      | # RandALCat preceded by LCat
-        \g<l_cat> .*? \g<r_and_al_cat>
-    /mux
+    Regexp.union(
+      /#{bidi_R_AL}.*?#{bidi_L}/mu, # RandALCat followed by LCat
+      /#{bidi_L}.*?#{bidi_R_AL}/mu, # RandALCat preceded by LCat
+    )
   end
 
   def bidi_fails_req3
-    / # contains RandALCat but doesn't start with RandALCat
-      \A(?<not_r_nor_al>#{bidi_not_R_AL})
-        .*?
-        (?<r_and_al_cat>#{bidi_R_AL})
-      | # contains RandALCat but doesn't end with RandALCat
-        \g<r_and_al_cat> .*? \g<not_r_nor_al>\z
-    /mux
+    # contains RandALCat:
+    Regexp.union(
+      /\A#{bidi_not_R_AL}.*?#{bidi_R_AL}/mu, # but doesn't start with RandALCat
+      /#{bidi_R_AL}.*?#{bidi_not_R_AL}\z/mu, # but doesn't end   with RandALCat
+    )
   end
 
-  # shares the bidi_R_AL definition between both req2 and req3
   def bidi_failure_regexp
-    req3_with_backref = bidi_fails_req3.source
-      .gsub(%r{\(\?\<r_and_al_cat\>\(.*?\)\)}, "\g<r_and_al_cat>")
-      .then{|re|"(?mx-i:#{re})"}
-    /#{bidi_fails_req2} | #{req3_with_backref}/mux
-  end
-
-  def bidi_consts
-    <<~RUBY
-      #############
-          # Bidirectional checks.
-          #
-
-    RUBY
+    Regexp.union(bidi_fails_req2, bidi_fails_req3)
   end
 
   SASL_TABLES_PROHIBITED = %w[

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: net-imap
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.5
 platform: ruby
 authors:
 - Shugo Maeda
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-26 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-protocol
@@ -76,34 +76,52 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/dependabot.yml"
+- ".github/workflows/pages.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- benchmarks/stringprep.yml
-- benchmarks/table-regexps.yml
 - docs/styles.css
 - lib/net/imap.rb
 - lib/net/imap/authenticators.rb
-- lib/net/imap/authenticators/cram_md5.rb
-- lib/net/imap/authenticators/digest_md5.rb
-- lib/net/imap/authenticators/login.rb
-- lib/net/imap/authenticators/plain.rb
-- lib/net/imap/authenticators/xoauth2.rb
 - lib/net/imap/command_data.rb
 - lib/net/imap/data_encoding.rb
+- lib/net/imap/deprecated_client_options.rb
 - lib/net/imap/errors.rb
+- lib/net/imap/fetch_data.rb
 - lib/net/imap/flags.rb
 - lib/net/imap/response_data.rb
 - lib/net/imap/response_parser.rb
+- lib/net/imap/response_parser/parser_utils.rb
 - lib/net/imap/sasl.rb
-- lib/net/imap/sasl/saslprep.rb
-- lib/net/imap/sasl/saslprep_tables.rb
+- lib/net/imap/sasl/anonymous_authenticator.rb
+- lib/net/imap/sasl/authentication_exchange.rb
+- lib/net/imap/sasl/authenticators.rb
+- lib/net/imap/sasl/client_adapter.rb
+- lib/net/imap/sasl/cram_md5_authenticator.rb
+- lib/net/imap/sasl/digest_md5_authenticator.rb
+- lib/net/imap/sasl/external_authenticator.rb
+- lib/net/imap/sasl/gs2_header.rb
+- lib/net/imap/sasl/login_authenticator.rb
+- lib/net/imap/sasl/oauthbearer_authenticator.rb
+- lib/net/imap/sasl/plain_authenticator.rb
+- lib/net/imap/sasl/protocol_adapters.rb
+- lib/net/imap/sasl/scram_algorithm.rb
+- lib/net/imap/sasl/scram_authenticator.rb
 - lib/net/imap/sasl/stringprep.rb
-- lib/net/imap/sasl/stringprep_tables.rb
+- lib/net/imap/sasl/xoauth2_authenticator.rb
+- lib/net/imap/sasl_adapter.rb
+- lib/net/imap/sequence_set.rb
+- lib/net/imap/stringprep.rb
+- lib/net/imap/stringprep/nameprep.rb
+- lib/net/imap/stringprep/saslprep.rb
+- lib/net/imap/stringprep/saslprep_tables.rb
+- lib/net/imap/stringprep/tables.rb
+- lib/net/imap/stringprep/trace.rb
 - net-imap.gemspec
+- rakelib/benchmarks.rake
 - rakelib/rdoc.rake
 - rakelib/rfcs.rake
 - rakelib/saslprep.rake
@@ -123,7 +141,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.7.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/benchmarks/stringprep.yml

@@ -1,65 +0,0 @@
----
-prelude: |
-  begin
-    require "mongo"  # gem install mongo
-    require "idn"    # gem install idn-ruby
-  rescue LoadError
-    warn "You must 'gem install mongo idn-ruby' for this benchmark."
-    raise
-  end
-
-  MStrPrep = Mongo::Auth::StringPrep
-
-  # this indirection will slow it down a little bit
-  def mongo_saslprep(string)
-    MStrPrep.prepare(string,
-                     MStrPrep::Profiles::SASL::MAPPINGS,
-                     MStrPrep::Profiles::SASL::PROHIBITED,
-                     normalize: true,
-                     bidi: true)
-  rescue Mongo::Error::FailedStringPrepValidation
-    nil
-  end
-
-  $LOAD_PATH.unshift "./lib"
-  require "net/imap"
-  def net_imap_saslprep(string)
-    Net::IMAP::SASL::SASLprep.saslprep string, exception: false
-  end
-
-  def libidn_saslprep(string)
-    IDN::Stringprep.with_profile(string, "SASLprep")
-  rescue IDN::Stringprep::StringprepError
-    nil
-  end
-
-benchmark:
-  - net_imap_saslprep "I\u00ADX"     # RFC example 1. IX
-  - net_imap_saslprep "user"         # RFC example 2. user
-  - net_imap_saslprep "USER"         # RFC example 3. user
-  - net_imap_saslprep "\u00aa"       # RFC example 4. a
-  - net_imap_saslprep "\u2168"       # RFC example 5. IX
-  - net_imap_saslprep "\u0007"       # RFC example 6. Error - prohibited character
-  - net_imap_saslprep "\u0627\u0031" # RFC example 7. Error - bidirectional check
-  - net_imap_saslprep "I\u2000X"     # map to space: I X
-  - net_imap_saslprep "a longer string, e.g. a password"
-
-  - libidn_saslprep "I\u00ADX"     # RFC example 1. IX
-  - libidn_saslprep "user"         # RFC example 2. user
-  - libidn_saslprep "USER"         # RFC example 3. user
-  - libidn_saslprep "\u00aa"       # RFC example 4. a
-  - libidn_saslprep "\u2168"       # RFC example 5. IX
-  - libidn_saslprep "\u0007"       # RFC example 6. Error - prohibited character
-  - libidn_saslprep "\u0627\u0031" # RFC example 7. Error - bidirectional check
-  - libidn_saslprep "I\u2000X"     # map to space: I X
-  - libidn_saslprep "a longer string, e.g. a password"
-
-  - mongo_saslprep "I\u00ADX"     # RFC example 1. IX
-  - mongo_saslprep "user"         # RFC example 2. user
-  - mongo_saslprep "USER"         # RFC example 3. user
-  - mongo_saslprep "\u00aa"       # RFC example 4. a
-  - mongo_saslprep "\u2168"       # RFC example 5. IX
-  - mongo_saslprep "\u0007"       # RFC example 6. Error - prohibited character
-  - mongo_saslprep "\u0627\u0031" # RFC example 7. Error - bidirectional check
-  - mongo_saslprep "I\u2000X"     # map to space: I X
-  - mongo_saslprep "a longer string, e.g. a password"

data/benchmarks/table-regexps.yml

@@ -1,39 +0,0 @@
-prelude: |
-  require "json"
-  require "set"
-
-  all_codepoints = (0..0x10ffff).map{_1.chr("UTF-8") rescue nil}.compact
-
-  rfc3454_tables = Dir["rfcs/rfc3454*.json"]
-    .first
-    .then{File.read _1}
-    .then{JSON.parse _1}
-  titles = rfc3454_tables.delete("titles")
-
-  sets = rfc3454_tables
-    .transform_values{|t|t.keys rescue t}
-    .transform_values{|table|
-      table
-        .map{_1.split(?-).map{|i|Integer i, 16}}
-        .flat_map{_2 ? (_1.._2).to_a : _1}
-        .to_set
-    }
-
-  TABLE_A1_SET   = sets.fetch "A.1"
-  ASSIGNED_3_2   = /\p{AGE=3.2}/
-  UNASSIGNED_3_2 = /\P{AGE=3.2}/
-  TABLE_A1_REGEX = /(?-mix:[\u{0000}-\u{001f}\u{007f}-\u{00a0}\u{0340}-\u{0341}\u{06dd}\u{070f}\u{1680}\u{180e}\u{2000}-\u{200f}\u{2028}-\u{202f}\u{205f}-\u{2063}\u{206a}-\u{206f}\u{2ff0}-\u{2ffb}\u{3000}\u{e000}-\u{f8ff}\u{fdd0}-\u{fdef}\u{feff}\u{fff9}-\u{ffff}\u{1d173}-\u{1d17a}\u{1fffe}-\u{1ffff}\u{2fffe}-\u{2ffff}\u{3fffe}-\u{3ffff}\u{4fffe}-\u{4ffff}\u{5fffe}-\u{5ffff}\u{6fffe}-\u{6ffff}\u{7fffe}-\u{7ffff}\u{8fffe}-\u{8ffff}\u{9fffe}-\u{9ffff}\u{afffe}-\u{affff}\u{bfffe}-\u{bffff}\u{cfffe}-\u{cffff}\u{dfffe}-\u{dffff}\u{e0001}\u{e0020}-\u{e007f}\u{efffe}-\u{10ffff}])|(?-mix:\p{Cs})/.freeze
-
-benchmark:
-
-  # matches A.1
-  - script: "all_codepoints.grep(TABLE_A1_SET)"
-  - script: "all_codepoints.grep(TABLE_A1_REGEX)"
-  - script: "all_codepoints.grep(UNASSIGNED_3_2)"
-  - script: "all_codepoints.grep_v(ASSIGNED_3_2)"
-
-  # doesn't match A.1
-  - script: "all_codepoints.grep_v(TABLE_A1_SET)"
-  - script: "all_codepoints.grep_v(TABLE_A1_REGEX)"
-  - script: "all_codepoints.grep_v(UNASSIGNED_3_2)"
-  - script: "all_codepoints.grep(ASSIGNED_3_2)"