icu 0.9.1 → 0.10.0

data/ext/icu/extconf.rb

@@ -0,0 +1,242 @@
+require 'mkmf'
+
+require 'rubygems'
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+
+# Utility functions
+
+def using_system_libraries?
+  arg_config('--use-system-libraries', !!ENV['ICU_USE_SYSTEM_LIBRARIES'])
+end
+
+# Building with system ICU
+
+if using_system_libraries?
+  message "Building ICU using system libraries.\n Not supported yet, PR welcome!"
+  exit 1
+
+  unless dir_config('icu').any?
+    base = if !`which brew`.empty?
+             `brew --prefix`.strip
+           elsif File.exists?("/usr/local/Cellar/icu4c")
+             '/usr/local/Cellar'
+           end
+
+    if base and icu4c = Dir[File.join(base, 'Cellar/icu4c/*')].sort.last
+      $INCFLAGS << " -I#{icu4c}/include "
+      $LDFLAGS  << " -L#{icu4c}/lib "
+    end
+  end
+
+  unless have_library 'icui18n' and have_header 'unicode/ucnv.h'
+    STDERR.puts <<-EOS
+************************************************************************
+icu not found.
+install by brew install icu4c or apt-get install libicu-dev)
+************************************************************************ww
+    EOS
+
+    exit(1)
+  end
+
+  have_library 'icuuc' or abort 'libicuuc missing'
+  have_library 'icudata' or abort 'libicudata missing'
+else
+  message "Building ICU from source.\n"
+
+  # The gem version constraint in the Rakefile is not respected at install time.
+  # Keep this version in sync with the one in the Rakefile !
+  require 'rubygems'
+  gem 'mini_portile2', '~> 2.2.0'
+  require 'mini_portile2'
+
+  # Checkout the source code of ICU.
+  # http://site.icu-project.org/download/
+  # http://userguide.icu-project.org/howtouseicu
+  # Also check the readme of ICU release file.
+  class ICURecipe < MiniPortile
+    def initialize(name, version, static_p)
+      super(name, version)
+      self.target = File.join(ROOT, "ports")
+      # Prefer host_alias over host in order to use i586-mingw32msvc as
+      # correct compiler prefix for cross build, but use host if not set.
+      self.host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"]
+      self.patch_files = Dir[File.join(ROOT, "patches", name, "*.patch")].sort
+      self.configure_options << "--libdir=#{File.join(self.path, "lib")}"
+
+      yield self
+
+      env = Hash.new do |hash, key|
+        hash[key] = ENV[key].dup.to_s rescue ''
+      end
+
+      self.configure_options.flatten!
+
+      self.configure_options.delete_if do |option|
+        case option
+          when /\A(\w+)=(.*)\z/
+            env[$1] = $2
+            true
+          else
+            false
+        end
+      end
+
+      if static_p
+        self.configure_options += [
+            "--enable-shared",
+            "--enable-static",
+            "--disable-renaming"
+        ]
+        env['CFLAGS'] = "-fPIC #{env['CFLAGS']}"
+        env['CPPFLAGS'] = "-DU_CHARSET_IS_UTF8=1 -DU_USING_ICU_NAMESPACE=0 -DU_STATIC_IMPLEMENTATION #{env['CPPFLAGS']}"
+        env['CXXFLAGS'] = "-fPIC -fno-exceptions #{env['CXXFLAGS']}"
+        env['LDFLAGS'] = "-fPIC -static-libstdc++ #{env['CFLAGS']}"
+      else
+        self.configure_options += [
+            "--enable-shared",
+            "--disable-static",
+        ]
+      end
+
+      if RbConfig::CONFIG['target_cpu'] == 'universal'
+        %w[CFLAGS LDFLAGS].each do |key|
+          unless env[key].include?('-arch')
+            env[key] += ' ' + RbConfig::CONFIG['ARCH_FLAG']
+          end
+        end
+      end
+
+      @env = env
+    end
+
+    def cook
+
+      message <<-"EOS"
+************************************************************************
+IMPORTANT NOTICE:
+
+Building ICU with a packaged version of #{name}-#{version}#{'.' if self.patch_files.empty?}
+      EOS
+
+      unless self.patch_files.empty?
+        message "with the following patches applied:\n"
+
+        self.patch_files.each do |patch|
+          message "\t- %s\n" % File.basename(patch)
+        end
+      end
+
+      message <<-"EOS"
+
+    gem install icu -- --use-system-libraries
+
+If you are using Bundler, tell it to use the option:
+
+    bundle config build.icu --use-system-libraries
+    bundle install
+      EOS
+
+      message <<-"EOS"
+************************************************************************
+      EOS
+      super
+    end
+
+    def configure
+      # run as recommend, basically set up compiler and flags
+      platform = if RUBY_PLATFORM =~ /mingw|mswin/
+                   'MSYS/MSVC'
+                 elsif RUBY_PLATFORM =~ /darwin/
+                   'MacOSX'
+                 else
+                   'Linux'
+                 end  # double quotes are significant.
+      execute('ICU Configure', [@env] + ['./runConfigureICU', platform] + computed_options)
+      super
+    end
+
+    def work_path
+      File.join(Dir.glob("#{tmp_path}/*").find { |d| File.directory?(d) }, 'source')
+    end
+
+  end
+
+  message "Using mini_portile version #{MiniPortile::VERSION}\n"
+
+  static_p = enable_config('static', true) or
+      message "Static linking is disabled.\n"
+  recipes = []
+
+  libicu_recipe = ICURecipe.new("libicu", "59.1", static_p) do |recipe|
+    recipe.files = [{
+                        url: "https://downloads.sourceforge.net/project/icu/ICU4C/59.1/icu4c-59_1-src.tgz?r=&ts=1501595646",
+                        sha256: "7132fdaf9379429d004005217f10e00b7d2319d0fea22bdfddef8991c45b75fe"
+                        # gpg: Signature made Fri Apr 14 21:00:23 2017 CEST using RSA key ID 4FB419E3
+                        # gpg: requesting key 4FB419E3 from hkps server hkps.pool.sks-keyservers.net
+                        # gpg: key 4FB419E3: public key "Steven R. Loomis (filfla-signing) <srloomis@us.ibm.com>" imported
+                        # gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model
+                        # gpg: depth: 0  valid:   2  signed:   1  trust: 0-, 0q, 0n, 0m, 0f, 2u
+                        # gpg: depth: 1  valid:   1  signed:   0  trust: 1-, 0q, 0n, 0m, 0f, 0u
+                        # gpg: next trustdb check due at 2018-08-19
+                        # gpg: Total number processed: 1
+                        # gpg:               imported: 1  (RSA: 1)
+                        # gpg: Good signature from "Steven R. Loomis (filfla-signing) <srloomis@us.ibm.com>" [unknown]
+                        # gpg:                 aka "Steven R. Loomis (filfla-signing) <srl295@gmail.com>" [unknown]
+                        # gpg:                 aka "Steven R. Loomis (filfla-signing) <srl@icu-project.org>" [unknown]
+                        # gpg:                 aka "[jpeg image of size 4680]" [unknown]
+                        # gpg: WARNING: This key is not certified with a trusted signature!
+                        # gpg:          There is no indication that the signature belongs to the owner.
+                        # Primary key fingerprint: BA90 283A 60D6 7BA0 DD91  0A89 3932 080F 4FB4 19E3
+                    }]
+  end
+  recipes.push libicu_recipe
+
+  recipes.each do |recipe|
+    checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed"
+    unless File.exist?(checkpoint)
+      recipe.cook
+      FileUtils.touch checkpoint
+    end
+
+    recipe.activate
+  end
+
+  $libs = $libs.shellsplit.tap do |libs|
+    [libicu_recipe].each do |recipe|
+      libname = recipe.name[/\Alib(.+)\z/, 1]
+      # TODO: build with pkg-config
+      # Should do like PKG_CONFIG_PATH=/root/icu4r/ports/x86_64-pc-linux-gnu/libicu/59.1/lib/pkgconfig/ pkg-config --static  icu-uc
+      File.join(recipe.path, "bin", "#{libname}-config").tap do |config|
+        # call config scripts explicit with 'sh' for compat with Windows
+        $CPPFLAGS = '-DU_DISABLE_RENAMING=1 -DU_CHARSET_IS_UTF8=1 -DU_USING_ICU_NAMESPACE=0 -DU_STATIC_IMPLEMENTATION' << ' ' << $CPPFLAGS
+        `sh #{config} --ldflags`.strip.shellsplit.each do |arg|
+          case arg
+            when /\A-L(.+)\z/
+              # Prioritize ports' directories
+              if $1.start_with?(ROOT + '/')
+                $LIBPATH = [$1] | $LIBPATH
+              else
+                $LIBPATH = $LIBPATH | [$1]
+              end
+            when /\A-l./
+              libs.unshift(arg)
+            else
+              $LDFLAGS << ' ' << arg.shellescape
+          end
+        end
+        $INCFLAGS = `sh #{config} --cppflags-searchpath `.strip << ' ' << $INCFLAGS
+        $CFLAGS = '-DU_DISABLE_RENAMING=1 -DU_CHARSET_IS_UTF8=1 -DU_USING_ICU_NAMESPACE=0 -DU_STATIC_IMPLEMENTATION' << ' ' << `sh #{config} --cflags`.strip << $CFLAGS
+      end
+    end
+  end.shelljoin
+
+end
+
+$CFLAGS << ' -O3 -funroll-loops -std=c99'
+$CFLAGS << ' -Wextra -O0 -ggdb3' if ENV['DEBUG']
+
+puts $CFLAGS, $CPPFLAGS, $CXXFLAGS
+
+create_makefile('icu/icu')

data/ext/icu/icu.c

@@ -0,0 +1,18 @@
+#include "icu.h"
+
+VALUE rb_mICU;
+
+void Init_icu(void)
+{
+    rb_mICU = rb_define_module("ICU");
+    init_internal_encoding();
+    init_rb_errors();
+    init_icu_collator();
+    init_icu_normalizer();
+    init_icu_spoof_checker();
+    init_icu_transliterator();
+    init_icu_charset_detector();
+    init_icu_locale();
+}
+
+/* vim: set expandtab sws=4 sw=4: */

data/ext/icu/icu.h

@@ -0,0 +1,78 @@
+#ifndef RUBY_EXTENSION_ICU_H_
+#define RUBY_EXTENSION_ICU_H_
+
+/* System libraries */
+#include <stdlib.h>
+
+/* Ruby headers */
+#define ONIG_ESCAPE_UCHAR_COLLISION 1  // ruby.h defines UChar macro
+#include <ruby.h>
+#include <ruby/encoding.h>
+#ifdef UChar // fail-safe
+  #undef UChar
+#endif
+#include "unicode/ustring.h"
+#include "unicode/uenum.h"
+#include "unicode/parseerr.h"
+
+/* Globals */
+
+extern VALUE rb_mICU;
+extern VALUE rb_eICU_Error;
+extern VALUE rb_eICU_InvalidParameterError;
+extern VALUE rb_cICU_UString;
+extern VALUE rb_cICU_Collator;
+extern VALUE rb_cICU_Normalizer;
+extern VALUE rb_cICU_SpoofChecker;
+extern VALUE rb_cICU_Transliterator;
+extern VALUE rb_cICU_CharsetDetector;
+extern VALUE rb_cICU_CharsetDetector_Match;
+extern VALUE rb_cICU_Locale;
+
+/* Prototypes */
+void Init_icu                                          _(( void ));
+void init_internal_encoding                            _(( void ));
+void init_rb_errors                                    _(( void ));
+void init_icu_collator                                 _(( void ));
+void init_icu_normalizer                               _(( void ));
+void init_icu_spoof_checker                            _(( void ));
+void init_icu_transliterator                           _(( void ));
+void init_icu_charset_detector                         _(( void ));
+void init_icu_locale                                   _(( void ));
+
+int icu_is_rb_enc_idx_as_utf_8                         _(( int ));
+int icu_is_rb_str_as_utf_8                             _(( VALUE ));
+const char* icu_rb_str_enc_name                        _(( int ));
+VALUE rb_str_enc_to_ascii_as_utf8                      _(( VALUE ));
+int icu_rb_str_enc_idx                                 _(( VALUE ));
+VALUE icu_enum_to_rb_ary                               _(( UEnumeration*, UErrorCode, long ));
+extern void icu_rb_raise_icu_error                     _(( UErrorCode ));
+extern void icu_rb_raise_icu_parse_error               _(( const UParseError* ));
+extern void icu_rb_raise_icu_invalid_parameter         _(( const char*, const char* ));
+
+VALUE icu_ustring_init_with_capa_enc                   _(( int32_t, int ));
+VALUE icu_ustring_from_rb_str                          _(( VALUE ));
+VALUE icu_ustring_from_uchar_str                       _(( const UChar*, int32_t ));
+void icu_ustring_clear_ptr                             _(( VALUE ));
+void icu_ustring_resize                                _(( VALUE, int32_t ));
+void icu_ustring_set_enc                               _(( VALUE, int ));
+VALUE icu_ustring_to_rb_enc_str_with_len               _(( VALUE, int32_t ));
+VALUE icu_ustring_to_rb_enc_str                        _(( VALUE ));
+UChar* icu_ustring_ptr                                 _(( VALUE ));
+int32_t icu_ustring_len                                _(( VALUE ));
+int32_t icu_ustring_capa                               _(( VALUE ));
+VALUE char_buffer_to_rb_str                            _(( const char* ));
+char* char_buffer_new                                  _(( int32_t ));
+void char_buffer_resize                                _(( const char*, int32_t ));
+void char_buffer_free                                  _(( const char* ));
+
+/* Constants */
+#define RUBY_C_STRING_TERMINATOR_SIZE 1
+
+/* Macros */
+#define ICU_RUBY_ENCODING_INDEX (rb_enc_to_index(rb_default_internal_encoding()) || rb_locale_encindex())
+#define ICU_RB_STRING_ENC_NAME_IDX(_idx) rb_enc_from_index(_idx) != NULL ? (rb_enc_from_index(_idx))->name : ""
+
+#endif // RUBY_EXTENSION_ICU_H_
+
+/* vim: set expandtab sws=4 sw=4: */

data/ext/icu/icu_charset_detector.c

@@ -0,0 +1,192 @@
+#include "icu.h"
+#include "unicode/ucsdet.h"
+
+#define GET_DETECTOR(_data) icu_detector_data* _data; \
+                            TypedData_Get_Struct(self, icu_detector_data, &icu_detector_type, _data)
+
+VALUE rb_cICU_CharsetDetector;
+VALUE rb_cICU_CharsetDetector_Match;
+
+typedef struct {
+    VALUE rb_instance;
+    UCharsetDetector* service;
+    char* dummy_str; // used for reset
+} icu_detector_data;
+
+static void detector_free(void* _this)
+{
+    icu_detector_data* this = _this;
+    if (this->dummy_str != NULL) {
+        ruby_xfree(this->dummy_str);
+    }
+    ucsdet_close(this->service);
+}
+
+static size_t detector_memsize(const void* _)
+{
+    return sizeof(icu_detector_data);
+}
+
+static const rb_data_type_t icu_detector_type = {
+    "icu/charset_detector",
+    {NULL, detector_free, detector_memsize,},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+static VALUE detector_populate_match_struct(const UCharsetMatch* match)
+{
+    UErrorCode status = U_ZERO_ERROR;
+    int32_t confidence = ucsdet_getConfidence(match, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    status = U_ZERO_ERROR;
+    const char* name = ucsdet_getName(match, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    status = U_ZERO_ERROR;
+    const char* language = ucsdet_getLanguage(match, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    return rb_struct_new(rb_cICU_CharsetDetector_Match,
+                         rb_str_new_cstr(name),
+                         INT2NUM(confidence),
+                         rb_str_new_cstr(language));
+}
+
+VALUE detector_alloc(VALUE self)
+{
+    icu_detector_data* this;
+    return TypedData_Make_Struct(self, icu_detector_data, &icu_detector_type, this);
+}
+
+VALUE detector_initialize(int argc, VALUE* argv, VALUE self)
+{
+    GET_DETECTOR(this);
+    this->rb_instance = self;
+    this->service = NULL;
+
+    UErrorCode status = U_ZERO_ERROR;
+    this->service = ucsdet_open(&status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    this->dummy_str = ALLOC_N(char, 1);
+    this->dummy_str[0] = '\0';
+
+    return self;
+}
+
+static inline void detector_reset_text(const icu_detector_data* this)
+{
+    UErrorCode status = U_ZERO_ERROR;
+    ucsdet_setText(this->service, this->dummy_str, 0, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+}
+
+// rb_str must be a ruby String
+static inline void detector_set_text(const icu_detector_data* this, VALUE rb_str)
+{
+    UErrorCode status = U_ZERO_ERROR;
+    ucsdet_setText(this->service, RSTRING_PTR(rb_str), RSTRING_LENINT(rb_str), &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+}
+
+//
+// no charset appears to match the data.
+// no input text has been provided
+VALUE detector_detect(VALUE self, VALUE str)
+{
+    StringValue(str);
+    GET_DETECTOR(this);
+
+    detector_set_text(this, str);
+    UErrorCode status = U_ZERO_ERROR;
+    const UCharsetMatch* match = ucsdet_detect(this->service, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+
+    VALUE rb_match = detector_populate_match_struct(match);
+    detector_reset_text(this);
+    return rb_match;
+}
+
+VALUE detector_detect_all(VALUE self, VALUE str)
+{
+    StringValue(str);
+    GET_DETECTOR(this);
+
+    detector_set_text(this, str);
+
+    UErrorCode status = U_ZERO_ERROR;
+    int32_t len_matches = 0;
+    const UCharsetMatch** matches = ucsdet_detectAll(this->service, &len_matches, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+
+    VALUE result = rb_ary_new2(3); // pre-allocate some slots
+    for (int32_t i = 0; i < len_matches; ++i) {
+        rb_ary_push(result, detector_populate_match_struct(matches[i]));
+    }
+    detector_reset_text(this);
+    return result;
+}
+
+static inline VALUE detector_get_input_filter_internal(const icu_detector_data* this)
+{
+    return ucsdet_isInputFilterEnabled(this->service) != 0 ? Qtrue : Qfalse;
+}
+
+VALUE detector_get_input_filter(VALUE self)
+{
+    GET_DETECTOR(this);
+    return detector_get_input_filter_internal(this);
+}
+
+VALUE detector_set_input_filter(VALUE self, VALUE flag)
+{
+    GET_DETECTOR(this);
+    ucsdet_enableInputFilter(this->service, flag == Qtrue ? TRUE : FALSE);
+    return detector_get_input_filter_internal(this);
+}
+
+VALUE detector_detectable_charsets(VALUE self)
+{
+    GET_DETECTOR(this);
+    UErrorCode status = U_ZERO_ERROR;
+    UEnumeration* charsets = ucsdet_getAllDetectableCharsets(this->service, &status);
+    return icu_enum_to_rb_ary(charsets, status, 28);
+}
+
+void init_icu_charset_detector(void)
+{
+    rb_cICU_CharsetDetector = rb_define_class_under(rb_mICU, "CharsetDetector", rb_cObject);
+    rb_define_alloc_func(rb_cICU_CharsetDetector, detector_alloc);
+    rb_define_method(rb_cICU_CharsetDetector, "initialize", detector_initialize, -1);
+    rb_define_method(rb_cICU_CharsetDetector, "detect", detector_detect, 1);
+    rb_define_method(rb_cICU_CharsetDetector, "detect_all", detector_detect_all, 1);
+    rb_define_method(rb_cICU_CharsetDetector, "input_filter", detector_get_input_filter, 0);
+    rb_define_method(rb_cICU_CharsetDetector, "input_filter=", detector_set_input_filter, 1);
+    rb_define_method(rb_cICU_CharsetDetector, "detectable_charsets", detector_detectable_charsets, 0);
+
+    // define a Match struct in Ruby
+    rb_cICU_CharsetDetector_Match = rb_struct_define_under(rb_cICU_CharsetDetector,
+                                                           "Match",
+                                                           "name",
+                                                           "confidence",
+                                                           "language",
+                                                           NULL);
+}
+
+#undef GET_DETECTOR
+
+/* vim: set expandtab sws=4 sw=4: */