utf8_proc 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51fd48320809134499eb6cc1ec7d84f91034df30
-  data.tar.gz: 8b25e229470de1b9e666d8bf55cebd4b647797ed
+  metadata.gz: b0f876c27547cb55eb9c0dd4ae3e4dc9686b76bd
+  data.tar.gz: ca58c8a85dbce81d92cbd724f38f29c81383349e
 SHA512:
-  metadata.gz: 03dee7d55aa9c396d079f1267033a3978ace36717c94b05a32d8225cda0e2c2d09f22e5f6fa058510b923796e948e031fabc48c6331b95d06a10ae0634400897
-  data.tar.gz: 2b3af9256deaa1cfea16eea4e6470e36edb37b091c8dd9e009d7e420e6ef50125e03f81c2921ee4ece325834f1c6dd698920ede15bbf7bb9de591713ea53ec65
+  metadata.gz: 56e03b1892309039f455c07cbc857076460a66bd910775df4eebfa95c597829731839ae2049d3006210531a85c3d2aa2720a0265997eb237782aba1e5b074163
+  data.tar.gz: f0a52ac5e29de9dc47c276ea8a4bda42a06579540331ff22dd8b89411f4a290d5c138894617b91f7756629fb7adc31ddf1d86be2912612232e0000c4c3f064c9

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/libutf8proc"]
+	path = vendor/libutf8proc
+	url = https://github.com/JuliaLang/utf8proc.git

data/.rubocop.yml

@@ -1,5 +1,7 @@
 AllCops:
   TargetRubyVersion: 2.0
+  Exclude:
+   - "vendor/**/*"
 
 # Metrics
 Metrics/AbcSize:

data/.travis.yml

@@ -15,8 +15,5 @@ matrix:
   - rvm: ruby-head
   - rvm: jruby-head
 before_install:
-  - wget -O utf8proc.zip https://github.com/JuliaLang/utf8proc/archive/v2.1.0.zip
-  - unzip utf8proc.zip
-  - pushd utf8proc* && sudo make install prefix=/usr && popd
-  - gem install bundler -v 1.14.4
+  - gem install bundler -v 1.14.5
 script: bundle exec rake

data/README.md

@@ -4,7 +4,7 @@
 [![Dependency Status](https://gemnasium.com/badges/github.com/nomoon/utf8_proc.svg)](https://gemnasium.com/github.com/nomoon/utf8_proc)
 [![Gem Version](https://badge.fury.io/rb/utf8_proc.svg)](https://badge.fury.io/rb/utf8_proc)
 
-A simple wrapper around [utf8proc](https://github.com/JuliaLang/utf8proc) for normalizing Unicode strings. Requires the `utf8proc` library and headers to be installed on your system. *(Packages are available. OSX: `brew install utf8proc`, Linux: `libutf8proc-dev` or `utf8proc-devel`)*
+A simple wrapper around [utf8proc](https://github.com/JuliaLang/utf8proc) for normalizing Unicode strings. Will use the `utf8proc` shared library and headers installed on your system if they are available *(Packages are available. OSX: `brew install utf8proc`, Linux: `libutf8proc-dev` or `utf8proc-devel`)*. Failing that, it will fall-back to compiling the library into the extension.
 
 Currently supports UTF-8/ASCII string input and NFC, NFD, NFKC, NFKD, and NKFC-Casefold forms. Handles Unicode 9.0 and includes the current official full suite of 9.0 normalization tests.
 
@@ -17,7 +17,7 @@ Quick benchmarks against the [UNF](https://github.com/knu/ruby-unf) gem show it
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'utf8_proc'
+gem "utf8_proc"
 ```
 
 And then execute:
@@ -53,6 +53,14 @@ UTF8Proc.normalize(utf8_string, form = :nfc)
 
 # Version string of loaded libutf8proc
 UTF8Proc::LIBRARY_VERSION
+
+# Add normalization methods directly to String class
+require "utf8_proc/core_ext/string"
+
+# This enables:
+"String".NFC
+"String".normalize(:nfc)
+
 ```
 
 (Like `unf`) on JRuby the gem will fall-back to using `java.text.normalizer`. The interface remains the same.

data/Rakefile

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 require "bundler/gem_tasks"
+require "rubocop/rake_task"
 require "rake/testtask"
 
+RuboCop::RakeTask.new
+
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-if defined?(JRUBY_VERSION)
-  task default: :test
+if RUBY_ENGINE == "jruby"
+  task default: %i[rubocop test]
 else
   require "rake/extensiontask"
 
@@ -19,5 +22,5 @@ else
     ext.lib_dir = "lib/utf8_proc"
   end
 
-  task default: %i[clobber compile test]
+  task default: %i[rubocop clobber compile test]
 end

data/ext/utf8_proc/extconf.rb

@@ -2,10 +2,19 @@
 # rubocop:disable GlobalVars
 require "mkmf"
 
-$CFLAGS << " -std=c99 -Wno-declaration-after-statement"
-
 pkg_config("utf8proc")
+unless have_library("utf8proc")
+  puts "Compiling local libutf8proc..."
+
+  libutf8proc_dir = File.expand_path(
+    File.join(File.dirname(__FILE__), "../../vendor/libutf8proc")
+  )
 
-have_library("utf8proc") || abort("This extension requires the utf8proc library.")
+  $VPATH << libutf8proc_dir
+  $srcs = ["utf8_proc.c", "utf8proc.c"]
+  $CFLAGS << " -I#{libutf8proc_dir}"
+end
+
+$CFLAGS << " -std=c99 -Wno-declaration-after-statement"
 
 create_makefile("utf8_proc/utf8_proc")

data/ext/utf8_proc/utf8_proc.c

@@ -32,29 +32,60 @@ static inline VALUE normInternal(VALUE string, utf8proc_option_t options) {
   return new_str;
 }
 
+// NFC
 
-VALUE toNFC(VALUE self, VALUE string) {
+static VALUE toNFC(VALUE self, VALUE string) {
   return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
 }
 
-VALUE toNFD(VALUE self, VALUE string) {
+static VALUE StoNFC(VALUE string) {
+  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
+}
+
+// NFD
+
+static VALUE toNFD(VALUE self, VALUE string) {
+  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
+}
+
+static VALUE StoNFD(VALUE string) {
   return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
 }
 
-VALUE toNFKC(VALUE self, VALUE string) {
+// NFKC
+
+static VALUE toNFKC(VALUE self, VALUE string) {
   return normInternal(string,UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
 }
 
-VALUE toNFKD(VALUE self, VALUE string) {
+static VALUE StoNFKC(VALUE string) {
+  return normInternal(string,UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+}
+
+// NFKD
+
+static VALUE toNFKD(VALUE self, VALUE string) {
+  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+}
+
+static VALUE StoNFKD(VALUE string) {
   return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
 }
 
-VALUE toNFKC_CF(VALUE self, VALUE string) {
+// NFKC_CF
+
+static VALUE toNFKC_CF(VALUE self, VALUE string) {
+  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
+}
+
+static VALUE StoNFKC_CF(VALUE string) {
   return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
 }
 
+// Parameterized normalization
 
-VALUE norm(int argc, VALUE* argv, VALUE self){
+
+static VALUE toNorm(int argc, VALUE* argv, VALUE self){
   VALUE string;
   VALUE form;
   rb_scan_args(argc, argv, "11", &string, &form);
@@ -82,6 +113,33 @@ VALUE norm(int argc, VALUE* argv, VALUE self){
   }
 }
 
+static VALUE StoNorm(int argc, VALUE* argv, VALUE string){
+  VALUE form;
+  rb_scan_args(argc, argv, "01", &form);
+
+  if (NIL_P(form)) {
+    return StoNFC(string);
+  }
+
+  ID s_form;
+  s_form = SYM2ID(form);
+  if (s_form == NFC) {
+    return StoNFC(string);
+  }else if(s_form == NFD) {
+    return StoNFD(string);
+  }else if(s_form == NFKC) {
+    return StoNFKC(string);
+  }else if(s_form == NFKD) {
+    return StoNFKD(string);
+  }else if(s_form == NFKC_CF) {
+    return StoNFKC_CF(string);
+  }else{
+    rb_raise(rb_eArgError, "%s",
+             "Argument must be one of [:nfc (default), :nfd, :nfkc, " \
+             ":nfkd, :nfkc_cf]");
+  }
+}
+
 void Init_utf8_proc(void) {
   VALUE rb_mBase;
   rb_mBase = rb_define_module("UTF8Proc");
@@ -105,5 +163,14 @@ void Init_utf8_proc(void) {
   rb_define_singleton_method(rb_mBase, "NFKC", toNFKC, 1);
   rb_define_singleton_method(rb_mBase, "NFKD", toNFKD, 1);
   rb_define_singleton_method(rb_mBase, "NFKC_CF", toNFKC_CF, 1);
-  rb_define_singleton_method(rb_mBase, "normalize", norm, -1);
+  rb_define_singleton_method(rb_mBase, "normalize", toNorm, -1);
+
+  VALUE rb_mStringExt;
+  rb_mStringExt = rb_define_module_under(rb_mBase, "StringExtension");
+  rb_define_method(rb_mStringExt, "NFC", StoNFC, 0);
+  rb_define_method(rb_mStringExt, "NFD", StoNFD, 0);
+  rb_define_method(rb_mStringExt, "NFKC", StoNFKC, 0);
+  rb_define_method(rb_mStringExt, "NFKD", StoNFKD, 0);
+  rb_define_method(rb_mStringExt, "NFKC_CF", StoNFKC_CF, 0);
+  rb_define_method(rb_mStringExt, "normalize", StoNorm, -1);
 }

data/lib/utf8_proc/core_ext/string.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "utf8_proc"
+
+class String
+  if RUBY_ENGINE == "jruby"
+    require "utf8_proc/core_ext/string_jruby"
+  else
+    include ::UTF8Proc::StringExtension
+  end
+end

data/lib/utf8_proc/core_ext/string_jruby.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+# rubocop:disable MethodName
+
+# This file should only be required within JRuby
+
+require "java"
+require "utf8_proc"
+
+class String
+  def NFC
+    ::UTF8Proc.NFC(self)
+  end
+
+  def NFD
+    ::UTF8Proc.NFD(self)
+  end
+
+  def NFKC
+    ::UTF8Proc.NFKC(self)
+  end
+
+  def NFKD
+    ::UTF8Proc.NFKD(self)
+  end
+
+  def NFKC_CF
+    ::UTF8Proc.NFKC_CF(self)
+  end
+
+  def normalize(form = :nfc)
+    ::UTF8Proc.normalize(self, form)
+  end
+end

data/lib/utf8_proc/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module UTF8Proc
-  VERSION = "0.3.0".freeze
+  VERSION = "0.4.0".freeze
 end

data/lib/utf8_proc.rb

@@ -2,7 +2,7 @@
 require "utf8_proc/version"
 
 module UTF8Proc
-  if defined?(JRUBY_VERSION)
+  if RUBY_ENGINE == "jruby"
     require "utf8_proc/jruby"
     include JRuby
   else

data/utf8_proc.gemspec

@@ -19,6 +19,10 @@ Gem::Specification.new do |spec|
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
+  spec.files += ["vendor/libutf8proc/utf8proc.c",
+                 "vendor/libutf8proc/utf8proc.h",
+                 "vendor/libutf8proc/utf8proc_data.c"]
+
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.14"
@@ -27,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest", "~> 5.10"
   spec.add_development_dependency "rubocop", "~> 0.47"
 
-  unless defined?(JRUBY_VERSION)
+  unless RUBY_ENGINE == "jruby"
     spec.extensions = ["ext/utf8_proc/extconf.rb"]
     spec.add_development_dependency "rake-compiler", "~> 1.0"
   end