unf_ext 0.0.8.2.beta-x64-mingw-ucrt

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0df2b2042b01091aaf309d44de895e1085657b92f1fe7f37c87beb41c0242f59
+  data.tar.gz: a9b406b2790051a65f2fedb2942bfecb6fa46836d25f096e1e5519380a4d57e8
+SHA512:
+  metadata.gz: 607c0bf6f8c11b205c7b755525ead44b5e90b21dcebfb90e0932b1f5ae09c79e0b92d8e804f8fec28c92296198bd7514097c98cb2b904bfac6a0886ccd6973ce
+  data.tar.gz: 12fc5c1d7f14828b9e07c4a986cd0f60084211685e6339ff63cdede0bab2bc41c9a1c1f774b207f7c3fbcffc774ed7a20ec88dd2d6635f3d77aebbdc132e720a

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.github/workflows/unf_ext.yml

@@ -0,0 +1,43 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: >-
+      ${{ matrix.os }} ${{ matrix.ruby }}
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-20.04, macos-11.0, windows-2019 ]
+        ruby: [ 2.6, 2.7, "3.0", 3.1, head ]
+        include:
+          - { os: windows-2019, ruby: mingw }
+        exclude:
+          - { os: windows-2019, ruby: head }
+
+    steps:
+      - name: repo checkout
+        uses: actions/checkout@v2
+
+      - name: load ruby cross-compilation toolkit
+        uses: MSP-Greg/setup-ruby-pkgs@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          mingw: _upgrade_
+
+      - name:  bundle install
+        shell: pwsh
+        run: |
+          bundle config set --local path .bundle/vendor
+          bundle install --jobs 4 --retry 3
+
+      - name: compile
+        timeout-minutes: 5
+        run: bundle exec rake compile
+
+      - name: test
+        timeout-minutes: 5
+        run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+*.so
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+ext/*/Makefile
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/CHANGELOG.md

@@ -0,0 +1,62 @@
+## 0.0.8.2 (2022-04-13)
+
+- Add x64-mingw-ucrt native gem support for RubyInstaller 3.1.
+
+## 0.0.8.1 (2022-03-13)
+
+- Include Windows binaries for Ruby 3.1. (FAIL)
+
+## 0.0.8 (2021-09-14)
+
+- No functional change in the library code.
+- Include Windows binaries for Ruby 3.0.
+- Drop support for Ruby 2.1 and earlier.
+- Replace Travis CI with Github Actions.
+- Fix cross-build after upgrading rake-compiler/rake-compiler-dock to 1.1.1/1.1.0.
+
+## 0.0.7.7 (2020-03-30)
+
+- Include Windows binaries for Ruby 2.7.
+
+## 0.0.7.6 (2019-03-19)
+
+- Include Windows binaries for Ruby 2.6.
+
+## 0.0.7.5 (2018-02-06)
+
+- Include Windows binaries for Ruby 2.5.
+
+## 0.0.7.4 (2017-04-19)
+
+- Fix build on ARM and GCC 6 again.
+
+## 0.0.7.3 (2017-04-11)
+
+- Update the base Unicode version to 9.
+
+- Fix compile issues on ARM and GCC 6.
+
+## 0.0.7.2 (2016-02-01)
+
+- Include Windows binaries for Ruby 2.3.
+
+## 0.0.7.1 (2015-04-18)
+
+- Windows fat binary gems no longer require libstd++ to
+  run, which are statically linked.
+
+- Add a fat binary gem for x64-mingw32 (64bit Windows).
+
+- Windows fat binary gems now include binaries for Ruby up to 2.2.
+
+## 0.0.6 (2013-02-16)
+
+- Migrate from Jeweler to Bundler.
+
+## 0.0.5 (2012-05-30)
+
+- Fix a type error for strict compilers.
+
+## 0.0.4 (2011-12-08)
+
+- Release under the current name of `unf_ext`.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ruby-unf_ext.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2010 Takeru Ohta <phjgt308@gmail.com>
+Copyright (c) 2011-2018 Akinori MUSHA <knu@idaemons.org> (extended Ruby support)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+ruby-unf_ext
+============
+
+Synopsis
+--------
+
+* Unicode Normalization Form support library for CRuby
+
+Description
+-----------
+
+* Normalizes UTF-8 strings into and from NFC, NFD, NFKC or NFKD
+
+        # For bulk conversion
+        normalizer = UNF::Normalizer.new
+        a_bunch_of_strings.map! { |string|
+          normalizer.normalize(string, :nfc) #=> string in NFC
+        }
+
+* Compliant with Unicode 9.0
+
+Requirement
+-----------
+
+* Ruby 1.8.7+, 1.9.2+
+
+* C++ compiler and libstdc++
+
+Installation
+------------
+
+	gem install unf_ext
+
+Or:
+
+    ruby extconf.rb && make && make install
+
+Development Resources
+---------------------
+
+* https://github.com/sile/unf
+
+    For issues regarding files under the directory `unf`, please
+    contact this upstream.
+
+* https://github.com/knu/ruby-unf_ext
+
+    The development site and the repository.
+
+License
+-------
+
+Copyright (c) 2010-2017 Takeru Ohta
+Copyright (c) 2011-2018 Akinori MUSHA
+
+Licensed under the MIT license.
+See `LICENSE` for details.

data/Rakefile

@@ -0,0 +1,37 @@
+require 'bundler/gem_tasks'
+
+gemspec = Bundler::GemHelper.gemspec
+
+native_platforms = %w[
+  x86-mingw32
+  x64-mingw32
+  x64-mingw-ucrt
+]
+
+require 'rake/extensiontask'
+Rake::ExtensionTask.new('unf_ext', gemspec) do |ext|
+  ext.cross_compile = true
+  ext.cross_platform = native_platforms
+  ext.cross_config_options << '--with-ldflags="-static-libgcc"' << '--with-static-libstdc++'
+end
+
+namespace :gem do
+  task :native do
+    require 'rake_compiler_dock'
+    sh 'bundle package --all'
+    native_platforms.each do |plat|
+      RakeCompilerDock.sh "bundle --local && rake native:#{plat} gem", platform: plat
+    end
+  end
+end
+
+task :gems => %i[build gem:native]
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.test_files = gemspec.test_files
+  test.verbose = true
+end
+
+task :default => :test

data/ext/unf_ext/extconf.rb

@@ -0,0 +1,50 @@
+require 'mkmf'
+
+if with_config('static-libstdc++')
+  $LDFLAGS << ' ' << `#{CONFIG['CC']} -print-file-name=libstdc++.a`.chomp
+else
+  have_library('stdc++')
+
+  case RbConfig::CONFIG['host_os']
+  when /solaris(!?2.11)/
+    # Do a little trickery here to enable C++ standard on Solaris 11 if found.
+    # This also forces 64bit compilation mode.
+    $CXX = CONFIG['CXX']
+    $CXX << ' ' << '-m64'
+    $CFLAGS = CONFIG['CFLAGS'].gsub(/-std=c99/, '')
+    $CFLAGS << ' ' << '-m64 -std=c++11'
+    $CPPFLAGS = CONFIG['CFLAGS'].gsub(/-std=c99/, '')
+    $CPPFLAGS << ' ' << '-m64 -std=c++11'
+    $CXXFLAGS = CONFIG['CFLAGS'].gsub(/-std=c99/, '')
+    $CXXFLAGS << ' ' << '-m64 -std=c++11'
+  when /aix/
+    # Compiler flags necessary on AIX.
+    # rubocop:disable Style/GlobalVars
+    $CFLAGS << ' ' << '-D_ALL_SOURCE=1'
+    $CPPFLAGS << ' ' << '-D_ALL_SOURCE=1'
+    $CXXFLAGS << ' ' << '-D_ALL_SOURCE=1'
+  end
+end
+
+create_makefile 'unf_ext'
+
+unless CONFIG['CXX']
+  case CONFIG['CC']
+  when %r{((?:.*[-/])?)gcc([-0-9.]*)$}
+    cxx = $1 + 'g++' + $2
+  when %r{((?:.*[-/])?)clang([-0-9.]*)$}
+    cxx = $1 + 'clang++' + $2
+  else
+    cxx = CONFIG['CC']
+  end
+
+  warn "CXX is automatically set to #{cxx}"
+
+  new_mf = <<-EOF << File.read('Makefile')
+CXX=#{cxx}
+  EOF
+
+  File.open('Makefile', 'w') { |mf|
+    mf.print new_mf
+  }
+end

data/ext/unf_ext/unf/normalizer.hh

@@ -0,0 +1,139 @@
+#ifndef UNF_NORMALIZER_HH
+#define UNF_NORMALIZER_HH
+
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <cstring>
+#include "trie/searcher.hh"
+#include "trie/char_stream.hh"
+#include "table.hh"
+#include "util.hh"
+
+namespace UNF {
+  class Normalizer {
+  public:
+    enum Form { FORM_NFD, FORM_NFC, FORM_NFKD, FORM_NFKC };
+
+  public:
+    Normalizer()
+      : nf_d(TABLE::NODES, TABLE::CANONICAL_DECOM_ROOT, (const char *)TABLE::STRINGS),
+	nf_kd(TABLE::NODES, TABLE::COMPATIBILITY_DECOM_ROOT, (const char *)TABLE::STRINGS),
+	nf_c(TABLE::NODES, TABLE::CANONICAL_COM_ROOT, (const char *)TABLE::STRINGS),
+	nf_c_qc(TABLE::NODES, TABLE::NFC_ILLEGAL_ROOT),
+	nf_kc_qc(TABLE::NODES, TABLE::NFKC_ILLEGAL_ROOT),
+	ccc(TABLE::NODES, TABLE::CANONICAL_CLASS_ROOT)
+    {}
+
+    const char* normalize(const char* src, Form form) {
+      switch(form) {
+      case FORM_NFD:  return nfd(src);
+      case FORM_NFC:  return nfc(src);
+      case FORM_NFKD: return nfkd(src);
+      case FORM_NFKC: return nfkc(src);
+      default:        return src;
+      }
+    }
+    const char* nfd(const char* src)  { return decompose(src, nf_d); }
+    const char* nfkd(const char* src) { return decompose(src, nf_kd); }
+    const char* nfc(const char* src)  { return compose(src, nf_c_qc, nf_d); }
+    const char* nfkc(const char* src) { return compose(src, nf_kc_qc, nf_kd); }
+
+  private:
+    const char* decompose(const char* src, const Trie::NormalizationForm& nf) {
+      const char* beg = next_invalid_char(src, nf);
+      if(*beg=='\0')
+	return src;
+      
+      buffer.assign(src, beg);
+      do {
+	const char* end = next_valid_starter(beg, nf);
+	decompose_one(beg, end, nf, buffer);
+	beg = next_invalid_char(end, nf);
+	buffer.append(end, beg);
+      } while(*beg!='\0');
+      
+      return buffer.c_str();      
+    }
+
+    void decompose_one(const char* beg, const char* end, const Trie::NormalizationForm& nf, std::string& buf) {
+      unsigned last = buf.size();
+      nf.decompose(Trie::RangeCharStream(beg,end), buf);
+      char* bufbeg = const_cast<char*>(buf.data());
+      canonical_combining_class_ordering(bufbeg+last, bufbeg+buf.size());
+    }
+
+    const char* compose(const char* src, const Trie::NormalizationForm& nf, const Trie::NormalizationForm& nf_decomp) {
+      const char* beg = next_invalid_char(src, nf);
+      if(*beg=='\0')
+	return src;
+      
+      buffer.assign(src, beg);
+      while(*beg!='\0') {
+	const char* end = next_valid_starter(beg, nf);
+	buffer2.clear();
+	decompose_one(beg, end, nf_decomp, buffer2);
+	end = compose_one(buffer2.c_str(), end, buffer);
+	beg = next_invalid_char(end, nf);
+	buffer.append(end, beg);
+      }
+
+      return buffer.c_str();      
+    }
+
+    const char* compose_one(const char* starter, const char* rest_starter, std::string& buf) {
+      Trie::CharStreamForComposition in(starter, rest_starter, canonical_classes, buffer3);
+      while(in.within_first())
+	nf_c.compose(in, buf);
+      return in.cur();
+    }
+
+    void canonical_combining_class_ordering(char* beg, const char* end) {
+      canonical_classes.assign(end-beg+1, 0); // +1 is for sentinel value
+      ccc.sort(beg, canonical_classes);
+    }
+
+    const char* next_invalid_char(const char* src, const Trie::NormalizationForm& nf) const {
+      int last_canonical_class = 0;
+      const char* cur = Util::nearest_utf8_char_start_point(src);
+      const char* starter = cur;
+      
+      for(; *cur != '\0'; cur = Util::nearest_utf8_char_start_point(cur+1)) {
+	int canonical_class = ccc.get_class(cur);
+	if(last_canonical_class > canonical_class && canonical_class != 0)
+	  return starter;
+
+	if(nf.quick_check(cur)==false)
+	  return starter;
+
+	if(canonical_class==0)
+	  starter=cur;
+
+	last_canonical_class = canonical_class;
+      }
+      return cur;
+    }
+
+    const char* next_valid_starter(const char* src, const Trie::NormalizationForm& nf) const {
+      const char* cur = Util::nearest_utf8_char_start_point(src+1);
+      while(ccc.get_class(cur)!=0 || nf.quick_check(cur)==false)
+	cur = Util::nearest_utf8_char_start_point(cur+1);
+      return cur;
+    }
+
+  private:
+    const Trie::NormalizationForm nf_d;
+    const Trie::NormalizationForm nf_kd;
+    const Trie::NormalizationForm nf_c;
+    const Trie::NormalizationForm nf_c_qc;
+    const Trie::NormalizationForm nf_kc_qc;
+    const Trie::CanonicalCombiningClass ccc;
+    
+    std::string buffer;
+    std::string buffer2;
+    std::string buffer3;
+    std::vector<unsigned char> canonical_classes;
+  };
+}
+
+#endif