utf8_proc 0.4.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0bcc124a9c3b7454adce5e3cbd9683bd23348df
-  data.tar.gz: 5f02f3653af737024f76ac89a4bbcd46fe1edc8a
+  metadata.gz: ef828b3bc3e2413f1ef0df107c8924eaa84cac4a
+  data.tar.gz: c4fe505d4cdb442a921df2bc3bd2d1bd0a08d08b
 SHA512:
-  metadata.gz: 556039eb22e607c1189a8ddd5b2ea3c9e248039403a6b1d08267002d01c979ce79e51c31aa96608e7c4c5b2a6b3872edc15adb0911846e49a9c40c86c29072fc
-  data.tar.gz: 2954531729a7da56ce6b035e1e7ce185da96accfdeea1e16f99fe2f53ee27ed1e14855f0d338a69105d0e1e606ef0fc8c92bf052d7c7b98cc4ad296b7d6a4a08
+  metadata.gz: ea8ec35886cf54a9ce7047ba26e66a0904d4171a26a2697a656dc4d746c3509a6123e7e926beb27f26960bd5d218ade3377b0c38efcc2cafb1acf2ce5ea612d6
+  data.tar.gz: 886eb31311b4d4f5559632d0347cd81ab6378d62285a958f0d2a2fcc193ae7f82aeefe945a192eee4d77d56ebdffa68426a0e4993890e83eec2bb305da3dbb5c

data/Gemfile

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 source "https://rubygems.org"
 
-# Eagerly load a version of the OpenSSL gem on MRI Ruby 2.4
-# Workaround for https://github.com/bundler/bundler/issues/5235
-if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION.start_with?("ruby 2.4")
-  gem "openssl"
-end
-
 # Specify your gem's dependencies in icu_test.gemspec
 gemspec

data/README.md

@@ -6,11 +6,9 @@
 
 A simple wrapper around [utf8proc](https://github.com/JuliaLang/utf8proc) for normalizing Unicode strings. Will use the `utf8proc` shared library and headers installed on your system if they are available *(Packages are available. OSX: `brew install utf8proc`, Linux: `libutf8proc-dev` or `utf8proc-devel`)*. Failing that, it will fall-back to compiling the library into the extension.
 
-Currently supports UTF-8/ASCII string input and NFC, NFD, NFKC, NFKD, and NKFC-Casefold forms. Handles Unicode 9.0 and includes the current official full suite of 9.0 normalization tests.
+Currently supports UTF-8/ASCII string input and NFC, NFD, NFKC, NFKD, and NKFC-Casefold forms (US-ASCII strings return an unmodified or case-folded copy). Handles Unicode 9.0 and includes the current official full suite of 9.0 normalization tests.
 
-Quick benchmarks against the [UNF](https://github.com/knu/ruby-unf) gem show it to be between the same speed (best-case) and ~2x slower (worst-case), averaging about ~1.2x slower on complex Unicode strings. The speed difference is more equal in NFC/NFD modes where mostly or already-normalized strings are used.
-
-*(Note: UNF is generally a bit faster but currently officially supports Unicode 6.0 and does not pass all 9.0 normalization tests.)*
+Quick benchmarks against the [UNF](https://github.com/knu/ruby-unf) gem show `utf8_proc` to be between the same speed (best-case) and ~2x slower (worst-case), averaging about 1.15 to 1.5x slower and improving on complex Unicode strings. *(However, UNF currently only officially supports Unicode 6.0 and does not pass all 9.0 normalization tests.)*
 
 ## Installation
 

data/ext/utf8_proc/extconf.rb

@@ -15,6 +15,10 @@ unless have_library("utf8proc")
   $CFLAGS << " -I#{libutf8proc_dir}"
 end
 
-$CFLAGS << " -std=c99 -Wno-declaration-after-statement"
+$CFLAGS << " -fPIC" if RbConfig::CONFIG["target_cpu"] == "x86_64"
+if RbConfig::CONFIG["CC"] == "clang"
+  $CFLAGS << " -flto -mllvm -inline-threshold=5000"
+end
+$CFLAGS << " -std=c99 -march=native -mtune=native -Wno-declaration-after-statement"
 
 create_makefile("utf8_proc/utf8_proc")

data/ext/utf8_proc/utf8_proc.c

@@ -9,104 +9,136 @@ static ID NFKC;
 static ID NFKD;
 static ID NFKC_CF;
 
-static inline void checkStrEncoding(VALUE *string) {
+// Derived from utf8proc_map_custom.
+// Pre-allocates string buffers and skips initial preflight decompose and
+// post-flight relloc for speed.
+static inline VALUE normInternal(VALUE *string, utf8proc_option_t options) {
   rb_encoding *enc;
   enc = rb_enc_get(*string);
-  if (enc != enc_utf8 && enc != enc_usascii) {
+  if (enc == enc_utf8) {
+    // Continue
+  } else if (enc == enc_usascii) {
+    // Return a US-ASCII copy, possibly case-folded
+    char *c_str = StringValuePtr(*string);
+    long c_strlen = RSTRING_LEN(*string);
+    if (options & UTF8PROC_CASEFOLD) {
+      for(int i = 0; i < c_strlen; i++) { c_str[i] = tolower(c_str[i]); }
+    }
+    return rb_enc_str_new(c_str, c_strlen, enc_usascii);
+  } else {
     rb_raise(rb_eEncodingError, "%s", "String must be in UTF-8 or US-ASCII encoding.");
   }
-}
 
-static inline VALUE normInternal(VALUE string, utf8proc_option_t options) {
-  checkStrEncoding(&string);
-  utf8proc_uint8_t *retval;
-  utf8proc_ssize_t retlen;
-  retlen = utf8proc_map(
-    (unsigned char *) StringValuePtr(string), RSTRING_LEN(string), &retval, options
-  );
+  // Allocate buffer to safe maximum length estimated from Normalization options
+  // and known maximal expansion of codepoints
+  utf8proc_int32_t *buffer;
+  utf8proc_ssize_t bufflen;
+  bufflen = rb_str_strlen(*string) * ((options & UTF8PROC_COMPAT) ? 18 : 4);
+  buffer = (utf8proc_int32_t *) malloc(bufflen * sizeof(utf8proc_int32_t) + 1);
+  if (!buffer) {
+    rb_raise(rb_eEncodingError, "%s", utf8proc_errmsg(UTF8PROC_ERROR_NOMEM));
+    return Qnil;
+  }
 
-  VALUE new_str;
-  new_str = rb_enc_str_new((char *) retval, retlen, rb_utf8_encoding());
-  free(retval);
+  utf8proc_ssize_t result;
+  // Decompose input
+  result = utf8proc_decompose_custom((unsigned char *) StringValuePtr(*string),
+                                     RSTRING_LEN(*string), buffer, bufflen,
+                                     options, NULL, NULL);
+  if (result < 0) {
+    free(buffer);
+    rb_raise(rb_eEncodingError, "%s", utf8proc_errmsg(result));
+    return Qnil;
+  }
 
+  // Compose & re-encode input.
+  result = utf8proc_reencode(buffer, result, options);
+  if (result < 0) {
+    free(buffer);
+    rb_raise(rb_eEncodingError, "%s", utf8proc_errmsg(result));
+    return Qnil;
+  }
+
+  VALUE new_str;
+  new_str = rb_enc_str_new((char *) buffer, result, enc_utf8);
+  free(buffer);
   return new_str;
 }
 
 // NFC
 
 static VALUE toNFC(VALUE self, VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
 }
 
 static VALUE StoNFC(VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
 }
 
 // NFD
 
 static VALUE toNFD(VALUE self, VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
 }
 
 static VALUE StoNFD(VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
 }
 
 // NFKC
 
 static VALUE toNFKC(VALUE self, VALUE string) {
-  return normInternal(string,UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
 }
 
 static VALUE StoNFKC(VALUE string) {
-  return normInternal(string,UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
 }
 
 // NFKD
 
 static VALUE toNFKD(VALUE self, VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
 }
 
 static VALUE StoNFKD(VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
 }
 
 // NFKC_CF
 
 static VALUE toNFKC_CF(VALUE self, VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
 }
 
 static VALUE StoNFKC_CF(VALUE string) {
-  return normInternal(string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
+  return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
 }
 
 // Parameterized normalization
 
-
 static VALUE toNorm(int argc, VALUE* argv, VALUE self){
   VALUE string;
   VALUE form;
   rb_scan_args(argc, argv, "11", &string, &form);
 
   if (NIL_P(form)) {
-    return toNFC(self, string);
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
   }
 
   ID s_form;
   s_form = SYM2ID(form);
   if (s_form == NFC) {
-    return toNFC(self, string);
-  }else if(s_form == NFD) {
-    return toNFD(self, string);
-  }else if(s_form == NFKC) {
-    return toNFKC(self, string);
-  }else if(s_form == NFKD) {
-    return toNFKD(self, string);
-  }else if(s_form == NFKC_CF) {
-    return toNFKC_CF(self, string);
-  }else{
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
+  } else if (s_form == NFD) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
+  } else if (s_form == NFKC) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  } else if (s_form == NFKD) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  } else if (s_form == NFKC_CF) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
+  } else {
     rb_raise(rb_eArgError, "%s",
              "Second argument must be one of [:nfc (default), :nfd, :nfkc, " \
              ":nfkd, :nfkc_cf]");
@@ -118,22 +150,22 @@ static VALUE StoNorm(int argc, VALUE* argv, VALUE string){
   rb_scan_args(argc, argv, "01", &form);
 
   if (NIL_P(form)) {
-    return StoNFC(string);
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
   }
 
   ID s_form;
   s_form = SYM2ID(form);
   if (s_form == NFC) {
-    return StoNFC(string);
-  }else if(s_form == NFD) {
-    return StoNFD(string);
-  }else if(s_form == NFKC) {
-    return StoNFKC(string);
-  }else if(s_form == NFKD) {
-    return StoNFKD(string);
-  }else if(s_form == NFKC_CF) {
-    return StoNFKC_CF(string);
-  }else{
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE);
+  } else if (s_form == NFD) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE);
+  } else if (s_form == NFKC) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  } else if (s_form == NFKD) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  } else if (s_form == NFKC_CF) {
+    return normInternal(&string, UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD);
+  } else {
     rb_raise(rb_eArgError, "%s",
              "Argument must be one of [:nfc (default), :nfd, :nfkc, " \
              ":nfkd, :nfkc_cf]");

data/ext/utf8_proc/utf8_proc.h

@@ -1,9 +1,12 @@
-#ifndef ICU_TEST_H
-#define ICU_TEST_H 1
+#ifndef UTF8_PROC_H
+#define UTF8_PROC_H 1
 
+#include <ctype.h>
 #include <ruby.h>
 #include <ruby/encoding.h>
 
 #include <utf8proc.h>
 
-#endif /* ICU_TEST_H */
+extern void Init_utf8_proc(void);
+
+#endif /* UTF8_PROC_H */

data/lib/utf8_proc.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require "utf8_proc/version"
+require "utf8_proc/benchmark"
 
 module UTF8Proc
   if RUBY_ENGINE == "jruby"

data/lib/utf8_proc/benchmark.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+# rubocop:disable MethodLength
+module UTF8Proc
+  module Benchmark
+    module_function
+
+    def run
+      require "benchmark/ips"
+      require "unf"
+      # Various different normalizations of Unicode characters.
+      test_arr = ["quick", "brown", "fox", "jumped", "over", "lazy", "dog",
+                  "QUICK", "BROWN", "FOX", "JUMPED", "OVER", "LAZY", "DOG",
+                  "\u{03D3}", "\u{03D2 0301}", "\u{038E}", "\u{03A5 0301}",
+                  "\u{03D4}", "\u{03D2 0308}", "\u{03AB}", "\u{03A5 0308}",
+                  "\u{1E9B}", "\u{017F 0307}", "\u{1E61}", "\u{0073 0307}",
+                  "\u{1D160}", "\u{1D158 1D165 1D16E}",
+                  "\u{1F82}", "\u{03B1 0313 0300 0345}",
+                  "\u{FDFA}", "\u{0635 0644 0649 0020 0627 0644 0644 0647}" \
+                  "\u{0020 0639 0644 064A 0647 0020 0648 0633 0644 0645}"] * 10
+      test_arr.concat([" "] * (test_arr.length / 4))
+
+      test_strings = Array.new(20) { test_arr.sample(30).join("") }
+      puts "\nBenchmark strings:\n\n * #{test_strings.join("\n * ")}\n\n"
+
+      ::Benchmark.ips do |x|
+        x.config(time: 10, warmup: 2)
+        x.report("UNF NFC") do
+          UNF::Normalizer.normalize(test_strings.sample, :nfc)
+        end
+
+        x.report("UTF8Proc NFC") do
+          UTF8Proc.normalize(test_strings.sample, :nfc)
+        end
+        x.compare!
+      end
+
+      ::Benchmark.ips do |x|
+        x.config(time: 10, warmup: 2)
+        x.report("UNF NFD") do
+          UNF::Normalizer.normalize(test_strings.sample, :nfd)
+        end
+
+        x.report("UTF8Proc NFD") do
+          UTF8Proc.normalize(test_strings.sample, :nfd)
+        end
+        x.compare!
+      end
+
+      ::Benchmark.ips do |x|
+        x.config(time: 10, warmup: 2)
+        x.report("UNF NFKC") do
+          UNF::Normalizer.normalize(test_strings.sample, :nfkc)
+        end
+
+        x.report("UTF8Proc NFKC") do
+          UTF8Proc.normalize(test_strings.sample, :nfkc)
+        end
+        x.compare!
+      end
+
+      ::Benchmark.ips do |x|
+        x.config(time: 10, warmup: 2)
+        x.report("UNF NFKD") do
+          UNF::Normalizer.normalize(test_strings.sample, :nfkd)
+        end
+
+        x.report("UTF8Proc NFKD") do
+          UTF8Proc.normalize(test_strings.sample, :nfkd)
+        end
+        x.compare!
+      end
+
+      ::Benchmark.ips do |x|
+        x.config(time: 10, warmup: 2)
+        x.report("UNF NFKC with .downcase") do
+          UNF::Normalizer.normalize(test_strings.sample, :nfkc).downcase
+        end
+
+        x.report("UTF8Proc NFKC_CF") do
+          UTF8Proc.normalize(test_strings.sample, :nfkc_cf)
+        end
+        x.compare!
+      end
+      true
+    end
+  end
+end

data/lib/utf8_proc/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module UTF8Proc
-  VERSION = "0.4.1".freeze
+  VERSION = "0.5.2".freeze
 end

data/utf8_proc.gemspec

@@ -1,5 +1,6 @@
 # coding: utf-8
 # frozen_string_literal: true
+# rubocop:disable BlockLength
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "utf8_proc/version"
@@ -17,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 2.0"
 
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+    f.match(%r{^(bin|test|spec|features)/})
   end
   spec.files += ["vendor/libutf8proc/LICENSE.md",
                  "vendor/libutf8proc/utf8proc.c",
@@ -31,6 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry", "~> 0.10"
   spec.add_development_dependency "minitest", "~> 5.10"
   spec.add_development_dependency "rubocop", "~> 0.47"
+  spec.add_development_dependency "benchmark-ips"
+  spec.add_development_dependency "unf"
 
   unless RUBY_ENGINE == "jruby"
     spec.extensions = ["ext/utf8_proc/extconf.rb"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: utf8_proc
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Tim Bellefleur
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-24 00:00:00.000000000 Z
+date: 2017-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.47'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
@@ -113,12 +141,11 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
 - ext/utf8_proc/extconf.rb
 - ext/utf8_proc/utf8_proc.c
 - ext/utf8_proc/utf8_proc.h
 - lib/utf8_proc.rb
+- lib/utf8_proc/benchmark.rb
 - lib/utf8_proc/core_ext/string.rb
 - lib/utf8_proc/core_ext/string_jruby.rb
 - lib/utf8_proc/jruby.rb

data/bin/console

@@ -1,12 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "utf8_proc"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-require "pry"
-Pry.start

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here