icu 0.9.1 → 0.10.0

data/ext/icu/icu_normalizer.c

@@ -0,0 +1,122 @@
+#include "icu.h"
+#include "unicode/unorm2.h"
+
+#define GET_NORMALIZER(_data) icu_normalizer_data* _data; \
+                              TypedData_Get_Struct(self, icu_normalizer_data, &icu_normalizer_type, _data)
+
+VALUE rb_cICU_Normalizer;
+static ID ID_nfc;
+static ID ID_nfkc;
+static ID ID_nfkc_cf;
+static ID ID_compose;
+static ID ID_decompose;
+
+typedef struct {
+    VALUE rb_instance;
+    int customized;
+    UNormalizer2* service;
+} icu_normalizer_data;
+
+static void normalizer_free(void* _this)
+{
+    icu_normalizer_data* this = _this;
+    if (this->customized == TRUE) { // If it's not constructed instance, shall not to be closed.
+        unorm2_close(this->service);
+    }
+}
+
+static size_t normalizer_memsize(const void* _)
+{
+    return sizeof(icu_normalizer_data);
+}
+
+static const rb_data_type_t icu_normalizer_type = {
+    "icu/normalizer",
+    {NULL, normalizer_free, normalizer_memsize,},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+VALUE normalizer_alloc(VALUE self)
+{
+    icu_normalizer_data* this;
+    return TypedData_Make_Struct(self, icu_normalizer_data, &icu_normalizer_type, this);
+}
+
+VALUE normalizer_initialize(int argc, VALUE* argv, VALUE self)
+{
+    VALUE sym_name;
+    VALUE sym_mode;
+    rb_scan_args(argc, argv, "02", &sym_name, &sym_mode);
+    if (NIL_P(sym_name)) {
+        sym_name = ID2SYM(ID_nfc);
+    }
+    if (NIL_P(sym_mode)) {
+        sym_mode = ID2SYM(ID_decompose);
+    }
+    int mode = UNORM2_DECOMPOSE;
+    if (sym_mode == ID2SYM(ID_compose)) {
+        mode = UNORM2_COMPOSE;
+    }
+    GET_NORMALIZER(this);
+    this->rb_instance = self;
+    this->customized = FALSE;
+
+    UErrorCode status = U_ZERO_ERROR;
+    this->service = (UNormalizer2*)unorm2_getInstance(NULL,
+                                                      rb_id2name(SYM2ID(sym_name)),
+                                                      mode,
+                                                      &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+
+    return self;
+}
+
+VALUE normalizer_normalize(VALUE self, VALUE rb_str)
+{
+    StringValue(rb_str);
+    GET_NORMALIZER(this);
+    VALUE in = icu_ustring_from_rb_str(rb_str);
+    VALUE out = icu_ustring_init_with_capa_enc(RSTRING_LENINT(rb_str) * 2 + RUBY_C_STRING_TERMINATOR_SIZE, ICU_RUBY_ENCODING_INDEX);
+
+    UErrorCode status = U_ZERO_ERROR;
+    int retried = FALSE;
+    int32_t len;
+    do {
+        len = unorm2_normalize(this->service,
+                               icu_ustring_ptr(in), icu_ustring_len(in),
+                               icu_ustring_ptr(out), icu_ustring_capa(out),
+                               &status);
+        if (!retried && status == U_BUFFER_OVERFLOW_ERROR) {
+            retried = TRUE;
+            icu_ustring_resize(out, len + RUBY_C_STRING_TERMINATOR_SIZE);
+            status = U_ZERO_ERROR;
+        } else if (U_FAILURE(status)) {
+            icu_rb_raise_icu_error(status);
+        } else { // retried == true && U_SUCCESS(status)
+            break;
+        }
+    } while (retried);
+
+    return icu_ustring_to_rb_enc_str_with_len(out, len);
+}
+
+void init_icu_normalizer(void)
+{
+    ID_nfc = rb_intern("nfc");
+    ID_nfkc = rb_intern("nfkc");
+    ID_nfkc_cf = rb_intern("nfkc_cf");
+    ID_compose = rb_intern("compose");
+    ID_decompose = rb_intern("decompose");
+
+    rb_cICU_Normalizer = rb_define_class_under(rb_mICU, "Normalizer", rb_cObject);
+    rb_define_alloc_func(rb_cICU_Normalizer, normalizer_alloc);
+    rb_define_method(rb_cICU_Normalizer, "initialize", normalizer_initialize, -1);
+    rb_define_method(rb_cICU_Normalizer, "normalize", normalizer_normalize, 1);
+}
+
+#undef GET_NORMALIZER
+
+/* vim: set expandtab sws=4 sw=4: */

data/ext/icu/icu_spoof_checker.c

@@ -0,0 +1,194 @@
+#include "icu.h"
+#include "unicode/uspoof.h"
+
+#define GET_SPOOF_CHECKER(_data) icu_spoof_checker_data* _data; \
+                                 TypedData_Get_Struct(self, icu_spoof_checker_data, &icu_spoof_checker_type, _data)
+
+VALUE rb_cICU_SpoofChecker;
+VALUE rb_mChecks;
+VALUE rb_mRestrictionLevel;
+
+typedef struct {
+    VALUE rb_instance;
+    USpoofChecker* service;
+} icu_spoof_checker_data;
+
+static void spoof_checker_free(void* _this)
+{
+    icu_spoof_checker_data* this = _this;
+    uspoof_close(this->service);
+}
+
+static size_t spoof_checker_memsize(const void* _)
+{
+    return sizeof(icu_spoof_checker_data);
+}
+
+static const rb_data_type_t icu_spoof_checker_type = {
+    "icu/spoof_checker",
+    {NULL, spoof_checker_free, spoof_checker_memsize,},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+VALUE spoof_checker_alloc(VALUE self)
+{
+    icu_spoof_checker_data* this;
+    return TypedData_Make_Struct(self, icu_spoof_checker_data, &icu_spoof_checker_type, this);
+}
+
+VALUE spoof_checker_initialize(VALUE self)
+{
+    GET_SPOOF_CHECKER(this);
+    this->rb_instance = self;
+    this->service = FALSE;
+
+    UErrorCode status = U_ZERO_ERROR;
+    this->service = uspoof_open(&status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+
+    return self;
+}
+
+static inline VALUE spoof_checker_get_restriction_level_internal(const icu_spoof_checker_data* this)
+{
+    URestrictionLevel level = uspoof_getRestrictionLevel(this->service);
+    return INT2NUM(level);
+}
+
+VALUE spoof_checker_get_restriction_level(VALUE self)
+{
+    GET_SPOOF_CHECKER(this);
+    return spoof_checker_get_restriction_level_internal(this);
+}
+
+VALUE spoof_checker_set_restriction_level(VALUE self, VALUE level)
+{
+    GET_SPOOF_CHECKER(this);
+    uspoof_setRestrictionLevel(this->service, NUM2INT(level));
+    return spoof_checker_get_restriction_level_internal(this);
+}
+
+static inline VALUE spoof_checker_get_checks_internal(const icu_spoof_checker_data* this)
+{
+    UErrorCode status = U_ZERO_ERROR;
+    int32_t checks = uspoof_getChecks(this->service, &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    return INT2NUM(checks);
+}
+
+VALUE spoof_checker_get_checks(VALUE self)
+{
+    GET_SPOOF_CHECKER(this);
+    return spoof_checker_get_checks_internal(this);
+}
+
+VALUE spoof_checker_set_checks(VALUE self, VALUE checks)
+{
+    GET_SPOOF_CHECKER(this);
+
+    UErrorCode status = U_ZERO_ERROR;
+    uspoof_setChecks(this->service, NUM2INT(checks), &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    return spoof_checker_get_checks_internal(this);
+}
+
+VALUE spoof_checker_confusable(VALUE self, VALUE str_a, VALUE str_b)
+{
+    StringValue(str_a);
+    StringValue(str_b);
+    GET_SPOOF_CHECKER(this);
+
+    VALUE tmp_a = icu_ustring_from_rb_str(str_a);
+    VALUE tmp_b = icu_ustring_from_rb_str(str_b);
+    UErrorCode status = U_ZERO_ERROR;
+    int32_t result = uspoof_areConfusable(this->service,
+                                          icu_ustring_ptr(tmp_a),
+                                          icu_ustring_len(tmp_a),
+                                          icu_ustring_ptr(tmp_b),
+                                          icu_ustring_len(tmp_b),
+                                          &status);
+
+    return INT2NUM(result);
+}
+
+VALUE spoof_checker_get_skeleton(VALUE self, VALUE str)
+{
+    StringValue(str);
+    GET_SPOOF_CHECKER(this);
+
+    VALUE in = icu_ustring_from_rb_str(str);
+    VALUE out = icu_ustring_init_with_capa_enc(icu_ustring_capa(in), ICU_RUBY_ENCODING_INDEX);
+    int retried = FALSE;
+    int32_t len_bytes;
+    UErrorCode status = U_ZERO_ERROR;
+    do {
+        // UTF-8 version does the conversion internally so we relies on UChar version here!
+        len_bytes = uspoof_getSkeleton(this->service, 0 /* deprecated */,
+                                       icu_ustring_ptr(in), icu_ustring_len(in),
+                                       icu_ustring_ptr(out), icu_ustring_capa(out),
+                                       &status);
+        if (!retried && status == U_BUFFER_OVERFLOW_ERROR) {
+            retried = TRUE;
+            icu_ustring_resize(out, len_bytes + RUBY_C_STRING_TERMINATOR_SIZE);
+            status = U_ZERO_ERROR;
+        } else if (U_FAILURE(status)) {
+            icu_rb_raise_icu_error(status);
+        } else { // retried == true && U_SUCCESS(status)
+            break;
+        }
+    } while (retried);
+
+    return icu_ustring_to_rb_enc_str_with_len(out, len_bytes);
+}
+
+#define DEFINE_SPOOF_ENUM_CONST(_MODULE, _NAME) rb_define_const(_MODULE, #_NAME, INT2NUM(USPOOF_##_NAME))
+
+void init_icu_spoof_checker(void)
+{
+    rb_cICU_SpoofChecker = rb_define_class_under(rb_mICU, "SpoofChecker", rb_cObject);
+    rb_mChecks = rb_define_module_under(rb_cICU_SpoofChecker, "Checks");
+    rb_mRestrictionLevel = rb_define_module_under(rb_cICU_SpoofChecker, "RestrictionLevel");
+    rb_include_module(rb_cICU_SpoofChecker, rb_mChecks);
+    rb_include_module(rb_cICU_SpoofChecker, rb_mRestrictionLevel);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, SINGLE_SCRIPT_CONFUSABLE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, MIXED_SCRIPT_CONFUSABLE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, WHOLE_SCRIPT_CONFUSABLE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, CONFUSABLE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, ANY_CASE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, RESTRICTION_LEVEL);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, SINGLE_SCRIPT);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, INVISIBLE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, CHAR_LIMIT);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, MIXED_NUMBERS);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, ALL_CHECKS);
+    DEFINE_SPOOF_ENUM_CONST(rb_mChecks, AUX_INFO);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, ASCII);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, SINGLE_SCRIPT_RESTRICTIVE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, HIGHLY_RESTRICTIVE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, MODERATELY_RESTRICTIVE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, MINIMALLY_RESTRICTIVE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, UNRESTRICTIVE);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, RESTRICTION_LEVEL_MASK);
+    DEFINE_SPOOF_ENUM_CONST(rb_mRestrictionLevel, UNDEFINED_RESTRICTIVE);
+
+    rb_define_alloc_func(rb_cICU_SpoofChecker, spoof_checker_alloc);
+    rb_define_method(rb_cICU_SpoofChecker, "initialize", spoof_checker_initialize, 0);
+    rb_define_method(rb_cICU_SpoofChecker, "restriction_level", spoof_checker_get_restriction_level, 0);
+    rb_define_method(rb_cICU_SpoofChecker, "restriction_level=", spoof_checker_set_restriction_level, 1);
+    rb_define_method(rb_cICU_SpoofChecker, "checks", spoof_checker_get_checks, 0);
+    rb_define_method(rb_cICU_SpoofChecker, "checks=", spoof_checker_set_checks, 1);
+    rb_define_method(rb_cICU_SpoofChecker, "confusable?", spoof_checker_confusable, 2);
+    rb_define_method(rb_cICU_SpoofChecker, "get_skeleton", spoof_checker_get_skeleton, 1);
+}
+
+#undef DEFINE_SPOOF_ENUM_CONST
+#undef GET_SPOOF_CHECKER
+
+/* vim: set expandtab sws=4 sw=4: */

data/ext/icu/icu_transliterator.c

@@ -0,0 +1,159 @@
+#include "icu.h"
+#include "unicode/utrans.h"
+
+#define GET_TRANSLITERATOR(_data) icu_transliterator_data* _data; \
+                                  TypedData_Get_Struct(self, icu_transliterator_data, &icu_transliterator_type, _data)
+
+VALUE rb_cICU_Transliterator;
+static ID ID_forward;
+static ID ID_reverse;
+
+typedef struct {
+    VALUE rb_instance;
+    UTransliterator* service;
+} icu_transliterator_data;
+
+static void transliterator_free(void* _this)
+{
+    icu_transliterator_data* this = _this;
+    utrans_close(this->service);
+}
+
+static size_t transliterator_memsize(const void* _)
+{
+    return sizeof(icu_transliterator_data);
+}
+
+static const rb_data_type_t icu_transliterator_type = {
+    "icu/transliterator",
+    {NULL, transliterator_free, transliterator_memsize,},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+VALUE transliterator_alloc(VALUE self)
+{
+    icu_transliterator_data* this;
+    return TypedData_Make_Struct(self, icu_transliterator_data, &icu_transliterator_type, this);
+}
+
+VALUE transliterator_initialize(int argc, VALUE* argv, VALUE self)
+{
+    GET_TRANSLITERATOR(this);
+    this->rb_instance = self;
+    this->service = NULL;
+
+    VALUE id;
+    VALUE direction;
+    VALUE rules;
+    rb_scan_args(argc, argv, "12", &id, &rules, &direction);
+    StringValue(id);
+    if (!NIL_P(rules)) {
+        StringValue(rules);
+    }
+    if (NIL_P(direction)) {
+        direction = ID2SYM(ID_forward);
+    } else {
+        // TODO: handle invalid direction
+    }
+
+    VALUE u_id = icu_ustring_from_rb_str(id);
+    UTransDirection u_direction = UTRANS_REVERSE;
+    if (SYM2ID(direction) == ID_forward) {
+        u_direction = UTRANS_FORWARD;
+    }
+    VALUE u_rules;
+    if (!NIL_P(rules)) {
+        u_rules = icu_ustring_from_rb_str(rules);
+    }
+    UParseError parser_error;
+    UErrorCode status = U_ZERO_ERROR;
+    this->service = utrans_openU(icu_ustring_ptr(u_id),
+                                        icu_ustring_len(u_id),
+                                        u_direction,
+                                        NIL_P(rules) ? NULL : icu_ustring_ptr(u_rules),
+                                        NIL_P(rules) ? 0 : icu_ustring_len(u_rules),
+                                        &parser_error, // TODO: should be possible to interpolate
+                                        &status);
+    if (U_FAILURE(status)) {
+        icu_rb_raise_icu_error(status);
+    }
+    if (this->service == NULL) {
+        rb_raise(rb_eICU_Error, "Transliterator can't be created.");
+    }
+
+    return self;
+}
+
+VALUE transliterator_transliterate(VALUE self, VALUE str)
+{
+    StringValue(str);
+    GET_TRANSLITERATOR(this);
+
+    VALUE u_str = icu_ustring_from_rb_str(str);
+    UErrorCode status = U_ZERO_ERROR;
+    int32_t original_len = icu_ustring_len(u_str);
+    int32_t capa = icu_ustring_capa(u_str);
+    int32_t len;
+    int32_t limit;
+    int retried = FALSE;
+    do {
+        len = limit = original_len;
+
+        utrans_transUChars(this->service,
+                           icu_ustring_ptr(u_str), &len, capa,
+                           0 /* always start from the beginning */, &limit,
+                           &status);
+
+        if (!retried && status == U_BUFFER_OVERFLOW_ERROR) {
+            retried = TRUE;
+            u_str = icu_ustring_from_rb_str(str);
+            capa = len + RUBY_C_STRING_TERMINATOR_SIZE;
+            icu_ustring_resize(u_str, capa);
+            status = U_ZERO_ERROR;
+        } else if (U_FAILURE(status)) {
+            icu_rb_raise_icu_error(status);
+        } else { // retried == true && U_SUCCESS(status)
+            break;
+        }
+    } while (retried);
+
+    return icu_ustring_to_rb_enc_str_with_len(u_str, len);
+}
+
+VALUE transliterator_unicode_id(VALUE self)
+{
+    GET_TRANSLITERATOR(this);
+
+    int32_t result_len = 0;
+    const UChar* result = utrans_getUnicodeID(this->service, &result_len);
+    VALUE str = icu_ustring_from_uchar_str(result, result_len);
+    VALUE rb_str = icu_ustring_to_rb_enc_str(str);
+    icu_ustring_clear_ptr(str);
+    return rb_str;
+}
+
+VALUE transliterator_available_ids(VALUE self)
+{
+    UErrorCode status = U_ZERO_ERROR;
+    UEnumeration* open_ids = utrans_openIDs(&status);
+    return icu_enum_to_rb_ary(open_ids, status, 650);
+}
+
+void init_icu_transliterator(void)
+{
+    ID_forward = rb_intern("forward");
+    ID_reverse = rb_intern("reverse");
+
+    rb_cICU_Transliterator = rb_define_class_under(rb_mICU, "Transliterator", rb_cObject);
+    rb_define_alloc_func(rb_cICU_Transliterator, transliterator_alloc);
+    rb_define_method(rb_cICU_Transliterator, "initialize", transliterator_initialize, -1);
+    rb_define_method(rb_cICU_Transliterator, "transliterate", transliterator_transliterate, 1);
+    rb_define_method(rb_cICU_Transliterator, "unicode_id", transliterator_unicode_id, 0);
+
+    rb_define_module_function(rb_cICU_Transliterator, "available_ids", transliterator_available_ids, 0);
+}
+
+#undef GET_TRANSLITERATOR
+
+/* vim: set expandtab sws=4 sw=4: */