ninjudd-icunicode 0.0.1

data/README.rdoc

@@ -0,0 +1,43 @@
+= ICUnicode
+
+Unicode sorting is complicated (http://unicode.org/reports/tr10), and Ruby doesn't do it
+correctly. But there is a widely-used implementation of the Unicode collation algorithm in
+the ICU (International Components for Unicode) libraries. There is also no way to do
+Transliteration in Ruby (http://userguide.icu-project.org/transforms/general). This gem is
+a simple C wrapper around ucol_getSortKey from the ICU Collation API and utrans_transUChars
+from the ICU Transliteration API. These are added as simple methods on String.
+
+== Usage:
+
+  ["cafe", "cafes", "caf\303\251"].sort
+  => ["cafe", "cafes", "caf\303\251"]
+   
+  require 'icunicode'
+
+  ["cafe", "cafes", "caf\303\251"].sort_by {|s| s.unicode_sort_key}
+  => ["cafe", "caf\303\251", "cafes"]
+
+  "blueberry".transliterate("Katakana").transliterate("Latin")
+  => "burueberrui"
+
+  "blueberry".transliterate("Greek").transliterate("Latin")
+  => "blyeberry"
+
+== Install:
+
+You must install ICU first. You can download the source from http://site.icu-project.org/download,
+or on Mac, you can install with MacPorts:
+
+  sudo port install icu
+
+Then install the gem:
+
+  sudo gem install ninjudd-icunicode -s http://gems.github.com
+
+== To do:
+
+Add support for locales other than en-US. Increase buffer size or make it grow dynamically.
+
+== License:
+
+Copyright (c) 2009 Justin Balthrop, Geni.com; Published under The MIT License, see LICENSE

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:minor: 0
+:patch: 1
+:major: 0

data/ext/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+have_library('icui18n')
+have_library('icuio')
+create_makefile('icunicode')

data/ext/icunicode.c

@@ -0,0 +1,85 @@
+#include "ruby.h"
+#include "unicode/ucol.h"
+#include "unicode/utrans.h"
+#include "unicode/ustring.h"
+#include "unicode/ustdio.h"
+ 
+#define BUF_SIZE 1000
+
+static void to_utf16(VALUE string, UChar *ustr, int32_t *ulen) {
+  UErrorCode status = U_ZERO_ERROR; 
+
+  string = StringValue(string);
+  u_strFromUTF8(ustr, BUF_SIZE, ulen, RSTRING_PTR(string), RSTRING_LEN(string), &status);
+  if (status == U_INVALID_CHAR_FOUND) ulen = 0;
+}
+
+static VALUE to_utf8(UChar *ustr, int32_t ulen) {
+  char str[BUF_SIZE];
+  int32_t len = 0;
+  UErrorCode status = U_ZERO_ERROR; 
+
+  u_strToUTF8(str, BUF_SIZE, &len, ustr, ulen, &status);
+  if (status == U_INVALID_CHAR_FOUND) len = 0;
+  return rb_str_new(str, len);
+}
+
+/*
+ * call-seq:
+ * string.unicode_sort_key -> string
+ *
+ * Returns a string that will sort according to the Unicode collation algorithm.
+ *
+ */
+static VALUE unicode_sort_key(VALUE string) {
+  char str[BUF_SIZE];
+  UChar ustr[BUF_SIZE];
+  int32_t len  = 0;
+  int32_t ulen = 0;
+  UErrorCode status = U_ZERO_ERROR; 
+  UCollator *col;
+
+  to_utf16(string, ustr, &ulen);
+
+  col = ucol_open("en_US", &status); 
+  if (U_SUCCESS(status)) {
+    len = ucol_getSortKey(col, ustr, ulen, (uint8_t*)str, BUF_SIZE);
+    ucol_close(col);
+  }    
+  
+  return rb_str_new(str, len - 1);
+}
+
+/*
+ * call-seq:
+ * string.transliterate(transform) -> string
+ *
+ * Transliterates string using transform.
+ *
+ */
+static VALUE unicode_transliterate(VALUE string, VALUE transform) {
+  UChar str[BUF_SIZE];
+  UChar trn[BUF_SIZE];
+  int32_t slen = 0;
+  int32_t tlen = 0;
+  UErrorCode  status = U_ZERO_ERROR; 
+  UTransliterator *trans;
+
+  to_utf16(string,    str, &slen);
+  to_utf16(transform, trn, &tlen);
+
+  trans = utrans_openU(trn, tlen, UTRANS_FORWARD, NULL, 0, NULL, &status);
+  if (trans) {
+    utrans_transUChars(trans, str, &slen, BUF_SIZE, 0, &slen, &status);
+    utrans_close(trans);
+  } else {
+    rb_raise(rb_eArgError, "invalid transform: %s", RSTRING_PTR(transform));
+  }
+
+  to_utf8(str, slen);
+}
+ 
+void Init_icunicode() {
+  rb_define_method(rb_cString, "unicode_sort_key", unicode_sort_key, 0);
+  rb_define_method(rb_cString, "transliterate", unicode_transliterate, 1);
+}

data/test/icunicode_test.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class UnicodeCollationTest < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'icunicode'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification 
+name: ninjudd-icunicode
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Justin Balthrop
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-15 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: ICU Unicode Transliteration and Collation in Ruby.
+email: code@justinbalthrop.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- README.rdoc
+- VERSION.yml
+- ext/icunicode.c
+- ext/extconf.rb
+- test/test_helper.rb
+- test/icunicode_test.rb
+has_rdoc: true
+homepage: http://github.com/ninjudd/unicode_collation
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: Unicode Transliteration and Collation in Ruby.
+test_files: []
+