icu 0.9.1 → 0.10.0

data/ext/icu/internal_encoding.c

@@ -0,0 +1,38 @@
+#include "icu.h"
+
+static rb_encoding* ascii_enc;
+static rb_encoding* utf8_enc;
+static ID ID_to_s;
+
+int icu_is_rb_enc_idx_as_utf_8(int enc_idx)
+{
+    return enc_idx == rb_ascii8bit_encindex() ||
+        enc_idx == rb_usascii_encindex() ||
+        enc_idx == rb_utf8_encindex();
+}
+
+int icu_is_rb_str_as_utf_8(VALUE str)
+{
+    return icu_is_rb_enc_idx_as_utf_8(ENCODING_GET(str));
+}
+
+int icu_rb_str_enc_idx(VALUE str)
+{
+    return ENCODING_GET(str);
+}
+
+VALUE rb_str_enc_to_ascii_as_utf8(VALUE str)
+{
+    str = rb_funcall(str, ID_to_s, 0);
+    // sanitize, will raise Encoding::UndefinedConversionError if anything wrong
+    str = rb_str_encode(str, rb_enc_from_encoding(ascii_enc), 0, Qnil);
+    rb_enc_associate(str, utf8_enc);
+    return str;
+}
+
+void init_internal_encoding(void)
+{
+    ascii_enc = rb_ascii8bit_encoding();
+    utf8_enc = rb_utf8_encoding();
+    ID_to_s = rb_intern("to_s");
+}

data/ext/icu/internal_ustring.c

@@ -0,0 +1,304 @@
+#include "icu.h"
+#include "unicode/ucnv.h"
+
+// #define ICU_USTRING_DEBUG 1
+
+#define GET_STRING(_data) icu_ustring_data* _data; \
+                          TypedData_Get_Struct(self, icu_ustring_data, &icu_ustring_type, _data)
+#define GET_STRING_VAL(_val, _data) icu_ustring_data* _data; \
+                                    TypedData_Get_Struct(_val, icu_ustring_data, &icu_ustring_type, _data)
+
+VALUE rb_cICU_UString;
+
+/* Data types */
+typedef struct {
+    int32_t len;
+    int32_t capa;
+    int rb_enc_idx;
+    UConverter* converter;
+    UChar* ptr;
+} icu_ustring_data;
+
+static void icu_ustring_free(void* _this)
+{
+    icu_ustring_data* this = _this;
+    ucnv_close(this->converter);
+    if (this->ptr != NULL) {
+        ruby_xfree(this->ptr);
+    }
+}
+
+static size_t icu_ustring_memsize(const void* data)
+{
+    icu_ustring_data* this = (icu_ustring_data*)data;
+    size_t size = sizeof(UChar) * this->capa;
+    return size + sizeof(icu_ustring_data);
+}
+
+static const rb_data_type_t icu_ustring_type = {
+    "icu/ustring",
+    {NULL, icu_ustring_free, icu_ustring_memsize,},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+// avoid name conflicts
+#include "ruby/encoding.h"
+
+/* Always allocate the internal string in a C function where you modify it
+ * as Ruby GC scans the C stacks and registers to find out GC root
+ */
+VALUE icu_ustring_alloc(VALUE self)
+{
+    icu_ustring_data* this;
+    return TypedData_Make_Struct(self, icu_ustring_data, &icu_ustring_type, this);
+}
+
+void icu_ustring_resize(VALUE self, int32_t capa)
+{
+    GET_STRING(this);
+    REALLOC_N(this->ptr, UChar, capa);
+    this->capa = capa;
+}
+
+// some ICU gives a pointer to buffer, used for releasing ptr since we don't manage it
+void icu_ustring_clear_ptr(VALUE self)
+{
+    GET_STRING(this);
+    this->ptr = NULL;
+}
+
+/*
+ Initialize the internal object. Used to convert string to ICU string.
+ See also:
+   - icu_ustring_init_with_capa_enc
+   - icu_ustring_from_uchar_str
+*/
+VALUE icu_ustring_from_rb_str(VALUE rb_str)
+{
+    StringValue(rb_str);
+    VALUE u_str = icu_ustring_alloc(rb_cICU_UString);
+    GET_STRING_VAL(u_str, this);
+    UErrorCode status = U_ZERO_ERROR;
+
+    this->rb_enc_idx = icu_rb_str_enc_idx(rb_str);
+    // take UTF-8 code path
+    if (icu_is_rb_enc_idx_as_utf_8(this->rb_enc_idx)) {
+        this->converter = NULL;
+    } else {
+        this->converter = ucnv_open(ICU_RB_STRING_ENC_NAME_IDX(this->rb_enc_idx), &status);
+        if (U_FAILURE(status)) {
+            icu_rb_raise_icu_error(status);
+        }
+    }
+
+    this->capa = RSTRING_LENINT(rb_str) + RUBY_C_STRING_TERMINATOR_SIZE;
+    this->ptr = ALLOC_N(UChar, this->capa);
+
+#ifdef ICU_USTRING_DEBUG
+    printf("icu_ustring_from_rb_str: %p %p %p %p %ld\n", u_str, this->ptr, rb_str, StringValuePtr(rb_str), RSTRING_LENINT(rb_str));
+#endif
+
+    status = U_ZERO_ERROR;
+    int retried = FALSE;
+    int32_t len;
+    do {
+        if (this->converter == NULL) {
+            u_strFromUTF8(this->ptr, this->capa, &len,
+                          RSTRING_PTR(rb_str), RSTRING_LENINT(rb_str),
+                          &status);
+        } else {
+            len = ucnv_toUChars(this->converter, this->ptr, this->capa,
+                                RSTRING_PTR(rb_str), RSTRING_LENINT(rb_str),
+                                &status);
+        }
+        if (!retried && status == U_BUFFER_OVERFLOW_ERROR) {
+            retried = TRUE;
+            this->capa = len + RUBY_C_STRING_TERMINATOR_SIZE;
+            REALLOC_N(this->ptr, UChar, this->capa);
+            status = U_ZERO_ERROR;
+        } else if (U_FAILURE(status)) {
+            icu_rb_raise_icu_error(status);
+        } else { // retried == true && U_SUCCESS(status)
+            break;
+        }
+    } while (retried);
+    this->len = len;
+
+    return u_str;
+}
+
+/*
+ Initialize the internal object. Used to convert string to Ruby string.
+ If the pointer points to a memory space managed by ICU, releasing it before the EOL of our object.
+ See also:
+   - icu_ustring_init_with_capa_enc
+   - icu_ustring_from_uchar_str
+   - icu_ustring_clear_ptr
+*/
+VALUE icu_ustring_from_uchar_str(const UChar* str, int32_t len)
+{
+    if (len <= 0) {
+        len = u_strlen(str);
+    }
+    VALUE u_str = icu_ustring_init_with_capa_enc(len + RUBY_C_STRING_TERMINATOR_SIZE, ICU_RUBY_ENCODING_INDEX);
+    GET_STRING_VAL(u_str, this);
+    this->ptr = (UChar*)str;
+    this->len = len;
+    return u_str;
+}
+
+/*
+ Initialize the internal object. Usually used as a buffer.
+ See also:
+   - icu_ustring_from_rb_str
+   - icu_ustring_from_uchar_str
+*/
+VALUE icu_ustring_init_with_capa_enc(int32_t capa, int enc_idx)
+{
+    VALUE buf = icu_ustring_alloc(rb_cICU_UString);
+    GET_STRING_VAL(buf, this);
+    icu_ustring_set_enc(buf, enc_idx);
+    this->capa = capa;
+    this->ptr = ALLOC_N(UChar, capa);
+    return buf;
+}
+
+void icu_ustring_set_enc(VALUE self, int enc_idx)
+{
+    GET_STRING(this);
+    if (this->converter != NULL) {
+        ucnv_close(this->converter);
+    }
+
+    this->rb_enc_idx = enc_idx;
+    // take UTF-8 code path
+    if (icu_is_rb_enc_idx_as_utf_8(enc_idx)) {
+        this->converter = NULL;
+    } else {
+        UErrorCode status = U_ZERO_ERROR;
+        this->converter = ucnv_open(ICU_RB_STRING_ENC_NAME_IDX(this->rb_enc_idx), &status);
+        if (U_FAILURE(status)) {
+            icu_rb_raise_icu_error(status);
+        }
+    }
+}
+
+VALUE icu_ustring_to_rb_enc_str_with_len(VALUE self, int32_t len)
+{
+    GET_STRING(this);
+
+#ifdef ICU_USTRING_DEBUG
+    printf("icu_ustring_to_rb_enc_str_with_len: %p %ld\n", self, len);
+#endif
+
+    this->len = len;
+    return icu_ustring_to_rb_enc_str(self);
+}
+
+VALUE icu_ustring_to_rb_enc_str(VALUE self)
+{
+    GET_STRING(this);
+    if (this->len < 0) {
+        this->len = 0;
+    }
+
+#ifdef ICU_USTRING_DEBUG
+    printf("icu_ustring_to_rb_enc_str: %p %d %d\n", self, this->len, this->capa);
+#endif
+
+    int32_t dest_len;
+    int32_t dest_capa = this->len + RUBY_C_STRING_TERMINATOR_SIZE;
+    char* dest = ALLOC_N(char, dest_capa);
+    int retried = FALSE;
+    UErrorCode status = U_ZERO_ERROR;
+    do {
+        if (this->converter == NULL) {
+            u_strToUTF8(dest, dest_capa, &dest_len, this->ptr, this->len, &status);
+        } else {
+            dest_len = ucnv_fromUChars(this->converter, dest, dest_capa,
+                                       this->ptr, this->len,
+                                       &status);
+        }
+        if (!retried && status == U_BUFFER_OVERFLOW_ERROR) {
+            retried = TRUE;
+            dest_capa = dest_len + RUBY_C_STRING_TERMINATOR_SIZE;
+            REALLOC_N(dest, char, dest_capa);
+            status = U_ZERO_ERROR;
+        } else if (U_FAILURE(status)) {
+            ruby_xfree(dest);
+            icu_rb_raise_icu_error(status);
+        } else { // retried == true && U_SUCCESS(status)
+            break;
+        }
+    } while (retried);
+
+#ifdef ICU_USTRING_DEBUG
+    printf("icu_ustring_to_rb_enc_str (before creating rb str): %p %d %d %d\n", (void *)self, this->len, this->capa, this->rb_enc_idx);
+    printf("icu_ustring_to_rb_enc_str (pointers): %p %p %d\n", (void *)self, dest, dest_len);
+#endif
+
+    VALUE rb_str = rb_enc_str_new(dest, dest_len, rb_enc_from_index(this->rb_enc_idx));
+    ruby_xfree(dest);
+    OBJ_TAINT(rb_str);
+    return rb_str;
+}
+
+
+inline UChar* icu_ustring_ptr_internal(const icu_ustring_data *this)
+{
+    return this->ptr;
+}
+
+inline UChar* icu_ustring_ptr(VALUE self)
+{
+    GET_STRING(this);
+    return icu_ustring_ptr_internal(this);
+}
+
+inline int32_t icu_ustring_len_internal(const icu_ustring_data *this)
+{
+    return this->len;
+}
+
+inline int32_t icu_ustring_len(VALUE self)
+{
+    GET_STRING(this);
+    return icu_ustring_len_internal(this);
+}
+
+inline int32_t icu_ustring_capa_internal(const icu_ustring_data *this)
+{
+    return this->capa;
+}
+
+inline int32_t icu_ustring_capa(VALUE self)
+{
+    GET_STRING(this);
+    return icu_ustring_capa_internal(this);
+}
+
+inline VALUE char_buffer_to_rb_str(const char* buffer)
+{
+    VALUE str = rb_str_new_cstr(buffer);
+    return rb_str_export_to_enc(str, rb_enc_from_index(ICU_RUBY_ENCODING_INDEX));
+}
+
+inline char* char_buffer_new(int32_t buf_size)
+{
+    char* buffer = ALLOC_N(char, buf_size);
+    return buffer;
+}
+
+inline void char_buffer_resize(const char* buffer, int32_t buf_size)
+{
+    REALLOC_N(buffer, char, buf_size);
+}
+
+inline void char_buffer_free(const char* buffer)
+{
+    ruby_xfree((void*)buffer);
+}
+
+#undef GET_STRING
+#undef GET_STRING_VAL

data/ext/icu/internal_utils.c

@@ -0,0 +1,50 @@
+#include "icu.h"
+
+VALUE icu_enum_to_rb_ary(UEnumeration* icu_enum, UErrorCode status, long pre_allocated)
+{
+    if (U_FAILURE(status)) {
+        uenum_close(icu_enum);
+        icu_rb_raise_icu_error(status);
+    }
+    VALUE result = rb_ary_new2(pre_allocated);
+    const UChar* ptr = NULL;
+    int32_t len = 0;
+    status = U_ZERO_ERROR;
+    while ((ptr = uenum_unext(icu_enum, &len, &status)) != NULL) {
+        if (U_FAILURE(status)) {
+            uenum_close(icu_enum);
+            icu_rb_raise_icu_error(status);
+        }
+        VALUE s = icu_ustring_from_uchar_str(ptr, len);
+        rb_ary_push(result, icu_ustring_to_rb_enc_str(s));
+        icu_ustring_clear_ptr(s);
+        status = U_ZERO_ERROR;
+    }
+    uenum_close(icu_enum);
+    return result;
+}
+
+extern inline void icu_rb_raise_icu_error(UErrorCode status)
+{
+    rb_raise(rb_eICU_Error, "ICU Error Code: %d, %s.", status, u_errorName(status));
+}
+
+
+extern inline void
+icu_rb_raise_icu_invalid_parameter(const char* parameter,
+                                   const char* error_message)
+{
+    rb_raise(rb_eICU_InvalidParameterError,
+             "ICU Invalid parameter: %s, %s.",
+             parameter,
+             error_message);
+}
+
+
+extern inline void icu_rb_raise_icu_parse_error(const UParseError* error)
+{
+    rb_raise(rb_eICU_InvalidParameterError,
+             "ICU Parse Error: Line %d, offset %d.",
+             error->line,
+             error->offset);
+}

data/ext/icu/rb_errors.c

@@ -0,0 +1,14 @@
+#include "icu.h"
+
+VALUE rb_eICU_Error;
+VALUE rb_eICU_InvalidParameterError;
+
+void init_rb_errors(void)
+{
+    rb_eICU_Error = rb_define_class_under(rb_mICU, "Error", rb_eStandardError);
+    rb_define_alias(rb_eICU_Error, "error", "message");
+    rb_eICU_InvalidParameterError = rb_define_class_under(rb_mICU, "InvalidParameterError", rb_eStandardError);
+    rb_define_alias(rb_eICU_InvalidParameterError, "error", "message");
+}
+
+/* vim: set expandtab sws=4 sw=4: */

data/icu.gemspec

@@ -0,0 +1,22 @@
+require File.expand_path('../lib/icu/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "icu"
+  s.version     = ICU::VERSION
+  s.authors     = ["Erick Guan"]
+  s.email       = ["fantasticfears@gmail.com"]
+  s.license     = "MIT"
+  s.homepage    = "https://github.com/fantasticfears/icu4r"
+  s.summary     = %q{A Unicode processing functions ruby gem, binding to ICU}
+  s.required_ruby_version = '>= 2.2.7'
+
+  s.extensions    = ["ext/icu/extconf.rb"]
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_development_dependency 'rake-compiler', ">= 0.7.5"
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'mini_portile2', '~> 2.2.0'
+end

data/lib/icu.rb

@@ -1,18 +1,6 @@
-# Copyright (c) 2007 Joshua Llorach
-# 
-# This file is part of ICU Ruby extension.
-#
-# ICU is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# ICU is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with ICU.  If not, see <http://www.gnu.org/licenses/>.
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'icu.so'))
+require 'icu/icu'
+require 'icu/version'
+require 'icu/collator'
+require 'icu/transliterator'
+require 'icu/charset_detector'
+require 'icu/locale'