unicode-data 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 84da723c97e50af6ce0b53902b81511f5333949d847d047046e070c9f640fe28
+  data.tar.gz: 477cb2a6ddf74b0b8ec8af07f3db680a205ef272b439da8978bc0b548fe20946
+SHA512:
+  metadata.gz: fb06a0321db3ffe4fbbeb459c9554bd65ef21bcf525a088778d7171234944fef797e64ee1ad9aefd6d5f3e31ca956e87c1d28ada4b730cb7f018ff72b6390f36
+  data.tar.gz: 5e10c86ca002f5240a3b80b026d32545c7a49b570270ae3b7875edb489907e060eedd9fdbc4ac90a5b65146db0a893e1b2534ce7d0f35781c8e23cc03e32f461

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# Ignore all of the derived data
+/lib/unicode/data/derived.txt

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    unicode-data (0.1.0)
+      rubyzip
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    minitest (5.14.4)
+    rake (13.0.6)
+    rubyzip (2.3.2)
+
+PLATFORMS
+  x86_64-darwin-19
+
+DEPENDENCIES
+  bundler
+  minitest
+  rake
+  unicode-data!
+
+BUNDLED WITH
+   2.2.24

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021-present Kevin Newton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# Unicode::Data
+
+A Ruby wrapping for the unicode character data set.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "unicode-data"
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install unicode-data
+
+## Usage
+
+When this gem is installed, it will automatically download the unicode character data set to a temporary zip file and generate a list of properties from that zip file. You can use this information (under `lib/unicode/data/derived.txt`) to implement, for example, a regular expression engine that can respect the unicode semantics defined by the [unicode technical standard](https://unicode.org/reports/tr18/). At the moment the list of properties generated includes:
+
+* General Categories
+* Blocks
+* Ages
+* Scripts
+* Script Extensions
+* Core Properties (Math, Alphabetic, Lowercase, Case_Ignorable, etc.)
+* Prop List Properties (White_Space, Bidi_Control, Terminal_Punctuation, etc.)
+
+This lines up to almost all of the [Onigmo](https://github.com/k-takata/Onigmo/blob/master/doc/UnicodeProps.txt) unicode support (and a lot more), with the exception of:
+
+* POSIX brackets
+* Emoji
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kddnewton/unicode-data.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("../../../lib", __dir__)
+$LOAD_PATH.unshift(lib)
+
+require "bundler/gem_tasks"
+require "rake/clean"
+require "rake/testtask"
+require "unicode/data"
+
+CLEAN.include(File.join(lib, "unicode/data/derived.txt"))
+
+namespace :ext do
+  load "ext/unicode/data/Rakefile"
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: :test
+
+namespace :"unicode:data" do
+  desc "Generate all of the neccesary derived files"
+  task :generate do
+    Unicode::Data.generate
+  end
+
+  desc "Validate all of the necessary derived files"
+  task :validate do
+    Unicode::Data.validate
+  end
+end

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "unicode/data"
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/ext/unicode/data/Rakefile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../../../lib", __dir__))
+require "unicode/data"
+
+desc "Download the unicode data file necessary for the current version of Ruby"
+task :default do
+  Unicode::Data.generate
+end

data/lib/unicode/data.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "unicode/data/version"
+
+module Unicode
+  module Data
+    def self.generate
+      require "unicode/data/generate"
+      Generate.call
+    end
+
+    def self.validate
+      require "unicode/data/validate"
+      Validate.call
+    end
+
+    def self.properties
+      @properties ||=
+        File.readlines(File.expand_path("data/derived.txt", __dir__), chomp: true).to_h do |line|
+          line.split(/\s+/, 2)
+        end
+    end
+
+    def self.property?(query, value)
+      properties[query].split(",").any? do |segment|
+        case segment
+        when /^(\d+)$/
+          $1.to_i == value.ord
+        when /^(\d+)..(\d+)$/
+          ($1.to_i..$2.to_i).cover?(value.ord)
+        end
+      end
+    end
+  end
+end

data/lib/unicode/data/generate.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require "logger"
+require "open-uri"
+require "zip"
+
+module Unicode
+  module Data
+    class Generate
+      class PropertyValueAliases
+        attr_reader :aliases
+
+        def initialize(aliases)
+          @aliases = aliases
+        end
+
+        def keys
+          aliases.keys
+        end
+
+        def find(property, value)
+          term = value.gsub(/[- ]/, "_")
+
+          aliases[property].find do |alias_set|
+            alias_set.any? { |alias_value| alias_value.casecmp(term) == 0 }
+          end
+        end
+      end
+
+      attr_reader :zipfile, :outfile, :logger
+
+      def initialize(zipfile, outfile, logger: Logger.new(STDOUT))
+        @zipfile = zipfile
+        @outfile = outfile
+        @logger = logger
+      end
+
+      def generate
+        property_aliases = read_property_aliases
+        property_value_aliases = PropertyValueAliases.new(read_property_value_aliases)
+
+        generate_general_categories
+        generate_blocks(property_value_aliases)
+        generate_ages(property_value_aliases)
+        generate_scripts(property_value_aliases)
+        generate_script_extensions(property_value_aliases)
+        generate_core_properties(property_aliases, property_value_aliases)
+        generate_prop_list_properties(property_aliases, property_value_aliases)
+      end
+
+      def self.call
+        unicode_version = RbConfig::CONFIG["UNICODE_VERSION"]
+
+        URI.open("https://www.unicode.org/Public/#{unicode_version}/ucd/UCD.zip") do |file|
+          Zip::File.open_buffer(file) do |zipfile|
+            File.open(File.join(__dir__, "derived.txt"), "w") do |outfile|
+              new(zipfile, outfile).generate
+            end
+          end
+        end
+      end
+
+      private
+
+      def each_line(filepath)
+        zipfile.get_input_stream(filepath).each_line do |line|
+          line.tap(&:chomp!).gsub!(/\s*#.*$/, "")
+          yield line unless line.empty?
+        end
+      end
+
+      def read_property_aliases
+        [].tap do |aliases|
+          each_line("PropertyAliases.txt") do |line|
+            aliases << line.split(/\s*;\s*/).uniq
+          end
+        end
+      end
+
+      def read_property_value_aliases
+        {}.tap do |aliases|
+          each_line("PropertyValueAliases.txt") do |line|
+            type, *values = line.split(/\s*;\s*/)
+            (aliases[type] ||= []) << values.uniq
+          end
+        end
+      end
+
+      GeneralCategory = Struct.new(:name, :abbrev, :aliased, :subsets, keyword_init: true)
+
+      # https://www.unicode.org/reports/tr44/#General_Category_Values
+      def generate_general_categories
+        properties = {}
+
+        zipfile.get_input_stream("PropertyValueAliases.txt").each_line do |line|
+          if line.start_with?("# General_Category") .. line.start_with?("# @missing")
+            match = /^gc ; (?<abbrev>[^\s]+)\s+; (?<name>[^\s]+)\s+(?:; (?<aliased>[^\s]+)\s+)?(?:\# (?<subsets>[^\s]+))?/.match(line)
+            next if match.nil?
+  
+            properties[match[:abbrev]] =
+              GeneralCategory.new(
+                name: match[:name],
+                abbrev: match[:abbrev],
+                aliased: match[:aliased],
+                subsets: match[:subsets]&.split(" | ")
+              )
+          end
+        end
+  
+        general_categories = read_property_codepoints("extracted/DerivedGeneralCategory.txt")
+        general_category_codepoints = {}
+
+        general_categories.each do |abbrev, codepoints|
+          general_category = properties[abbrev]
+
+          queries = [abbrev, general_category.name]
+          queries << general_category.aliased if general_category.aliased
+          queries.map! { |value| "\\p{General_Category=#{value}}" }
+
+          if general_category.subsets
+            codepoints =
+              general_category.subsets.flat_map do |subset|
+                general_categories[subset]
+              end
+          end
+
+          general_category_codepoints[abbrev] = codepoints
+          write_queries(queries, codepoints)
+        end
+
+        # https://unicode.org/reports/tr18/#General_Category_Property  
+        # There are a couple of special categories that are defined that we will
+        # handle here.
+        write_queries(["\\p{Any}"], [0..0x10FFFF])
+        write_queries(["\\p{Assigned}"], (0..0x10FFFF).to_a - general_category_codepoints["Cn"].flat_map { |codepoint| [*codepoint] })
+        write_queries(["\\p{ASCII}"], [0..0x7F])
+      end
+
+      def generate_blocks(property_value_aliases)
+        read_property_codepoints("Blocks.txt").each do |block, codepoints|
+          write_queries(
+            property_value_aliases.find("blk", block).map { |value| "\\p{Block=#{value}}" },
+            codepoints
+          )
+        end
+      end
+
+      # https://www.unicode.org/reports/tr44/#Character_Age
+      def generate_ages(property_value_aliases)
+        ages = read_property_codepoints("DerivedAge.txt").to_a
+        ages.each_with_index do |(version, _values), index|
+          # When querying by age, something that was added in 1.1 will also
+          # match at \p{age=2.0} query, so we need to get every value from all
+          # of the preceeding ages as well.
+          write_queries(
+            property_value_aliases.find("age", version).map { |value| "\\p{Age=#{value}}" },
+            ages[0..index].flat_map(&:last)
+          )
+        end
+      end
+
+      # https://www.unicode.org/reports/tr24/
+      def generate_scripts(property_value_aliases)
+        read_property_codepoints("Scripts.txt").each do |script, codepoints|
+          write_queries(
+            property_value_aliases.find("sc", script).map { |value| "\\p{Script=#{value}}" },
+            codepoints
+          )
+        end
+      end
+
+      def generate_script_extensions(property_value_aliases)
+        script_extensions = {}
+
+        read_property_codepoints("ScriptExtensions.txt").each do |script_extension_set, codepoints|
+          script_extension_set.split(" ").each do |script_extension|
+            script_extensions[script_extension] ||= []
+            script_extensions[script_extension] += codepoints
+          end
+        end
+
+        script_extensions.each do |script_extension, codepoints|
+          write_queries(
+            property_value_aliases.find("sc", script_extension)
+              .map { |value| "\\p{Script_Extensions=#{value}}" },
+            codepoints
+          )
+        end
+      end
+
+      def generate_core_properties(property_aliases, property_value_aliases)
+        read_property_codepoints("DerivedCoreProperties.txt").each do |property, codepoints|
+          property_alias_set =
+            property_aliases.find { |alias_set| alias_set.include?(property) }
+
+          property_value_alias_key =
+            (property_alias_set & property_value_aliases.keys).first
+
+          write_queries(["\\p{#{property_value_alias_key}=True}"], codepoints)
+        end
+      end
+
+      def generate_prop_list_properties(property_aliases, property_value_aliases)
+        read_property_codepoints("PropList.txt").each do |property, codepoints|
+          property_alias_set =
+            property_aliases.find { |alias_set| alias_set.include?(property) }
+
+          property_value_alias_key =
+            (property_alias_set & property_value_aliases.keys).first
+
+          write_queries(["\\p{#{property_value_alias_key}=True}"], codepoints)
+        end
+      end
+
+      def read_property_codepoints(filepath)
+        {}.tap do |properties|
+          each_line(filepath) do |line|
+            codepoint, property = line.split(/\s*;\s*/)
+            codepoint =
+              if codepoint.include?("..")
+                left, right = codepoint.split("..").map { |value| value.to_i(16) }
+                left..right
+              else
+                codepoint.to_i(16)
+              end
+
+            (properties[property] ||= []) << codepoint
+          end
+        end
+      end
+
+      def write_queries(queries, codepoints)
+        serialized =
+          codepoints
+            .flat_map { |codepoint| [*codepoint] }
+            .sort
+            .chunk_while { |prev, curr| curr - prev == 1 }
+            .map { |chunk| chunk.length > 1 ? "#{chunk[0]}..#{chunk[-1]}" : chunk[0] }
+            .join(",")
+
+        queries.each do |query|
+          logger.info("Generating #{query}")
+          outfile.puts("%-80s %s" % [query, serialized])
+        end
+      end
+    end
+  end
+end

data/lib/unicode/data/validate.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module Unicode
+  module Data
+    class Validate
+      module Mode
+        # Just test the first value in the range of characters
+        class First
+          def apply(values, &block)
+            block.call(values.first)
+          end
+        end
+
+        # Test a sample of 50 random values from the range of characters
+        class Sample
+          def apply(values, &block)
+            values.to_a.sample(50).each(&block)
+          end
+        end
+
+        # Test every value from the range of characters
+        class Full
+          def apply(values, &block)
+            values.each(&block)
+          end
+        end
+      end
+
+      attr_reader :logger, :mode, :surrogates
+
+      def initialize(logger: Logger.new(STDOUT), mode: ENV.fetch("MODE", "first"))
+        @logger = logger
+        @mode =
+          case mode
+          when "first"  then Mode::First.new
+          when "sample" then Mode::Sample.new
+          when "full"   then Mode::Full.new
+          else
+            raise ArgumentError, "invalid mode: #{mode}"
+          end
+
+        # This is a list of all of the surrogate characters that exist so that
+        # we can skip them when validating since they're not valid in UTF-8.
+        File.foreach(File.join(__dir__, "derived.txt"), chomp: true) do |line|
+          property, values = line.split(" ", 2)
+
+          if property.start_with?("\\p{General_Category=Surrogate}")
+            @surrogates = each_value(values, Mode::Full.new).to_a
+            break
+          end
+        end
+      end
+
+      def validate
+        File.foreach(File.join(__dir__, "derived.txt"), chomp: true) do |line|
+          property, values = line.split(/\s+/, 2)
+
+          # For general categories and scripts, we don't actually want the
+          # prefix in the property name, so here leave it out.
+          property.gsub!(/(General_Category|Script)=/, "")
+
+          # Ruby doesn't support Block= syntax, it expects you to instead have
+          # no property name and have the block name begin with In_.
+          property.gsub!(/Block=/, "In_")
+
+          # Ruby doesn't support boolean property querying with values, it only
+          # supports the plain property name.
+          property.gsub!(/=(Yes|Y|True|T)/, "")
+
+          pattern =
+            begin
+              /#{property}/
+            rescue RegexpError
+              # There are a fair amount of properties that we have in this gem
+              # that Ruby doesn't support natively. Things like aliases for the
+              # various blocks, script extensions, aliases for the ages, etc.
+              # In this case just rescue the error and move on since we can't
+              # validate against native.
+              logger.warn("Skipping   #{property}")
+              next
+            end
+
+          logger.info("Validating #{property}")
+
+          each_value(values, mode) do |value|
+            next if surrogates.include?(value)
+            raise unless pattern.match?([value].pack("U"))
+          end
+        end
+      end
+
+      def self.call
+        new.validate
+      end
+
+      private
+
+      def each_value(values, mode, &block)
+        return enum_for(__method__, values, mode) unless block_given?
+
+        values.split(",").each do |value|
+          case value
+          when /^(\d+)$/
+            block.call($1.to_i)
+          when /^(\d+)..(\d+)$/
+            mode.apply($1.to_i..$2.to_i, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unicode/data/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Unicode
+  module Data
+    VERSION = "0.1.0"
+  end
+end

data/unicode-data.gemspec

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "unicode/data/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "unicode-data"
+  spec.version       = Unicode::Data::VERSION
+  spec.authors       = ["Kevin Newton"]
+  spec.email         = ["kddnewton@gmail.com"]
+
+  spec.summary       = "A Ruby port of the unicode character data"
+  spec.homepage      = "https://github.com/kddnewton/unicode-data"
+  spec.license       = "MIT"
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/unicode/data/Rakefile"]
+
+  spec.add_dependency "rubyzip"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: unicode-data
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kevin Newton
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- kddnewton@gmail.com
+executables: []
+extensions:
+- ext/unicode/data/Rakefile
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ext/unicode/data/Rakefile
+- lib/unicode/data.rb
+- lib/unicode/data/generate.rb
+- lib/unicode/data/validate.rb
+- lib/unicode/data/version.rb
+- unicode-data.gemspec
+homepage: https://github.com/kddnewton/unicode-data
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: A Ruby port of the unicode character data
+test_files: []