unicode 0.1

data/README

@@ -0,0 +1,107 @@
+		   Unicode Library for Ruby
+			Version 0.1
+
+		       Yoshida Masato
+
+
+- Introduction
+
+  Unicode string manipulation library for Ruby.
+  This library is based on UTR #15 Unicode Normalization Forms(*1).
+
+    *1 <URL:http://www.unicode.org/unicode/reports/tr15/>
+
+
+- Install
+
+  This can work with ruby-1.4 or later. I recommend you to
+  use ruby-1.4.2 or later.
+
+  Make and install usually.
+  For example, when Ruby supports dynamic linking on your OS,
+
+    ruby extconf.rb
+    make
+    make install
+
+
+- Usage
+
+  If you do not link this module with Ruby statically, 
+
+    require "unicode"
+
+  before using.
+
+
+- Module Functions
+
+  All parameters of functions must be UTF-8.
+
+  Unicode::strcmp(str1, str2)
+  Unicode::strcmp_compat(str1, str2)
+    Compares Unicode strings with normalization.
+    strcmp uses Normalization Form D, strcmp_compat uses
+    Normalization Form KD.
+
+  Unicode::decopose(str)
+  Unicode::decopose_compat(str)
+    Decompose Unicode string. Then the trailing characters
+    are sorted in canonical order.
+    decompose uses the canonical decomposition,
+    decompose_compat uses the compatibility decomposition.
+    The decomposition is based on the character decomposition
+    mapping in UnicodeData.txt and the Hangul decomposition
+    algorithm.
+
+  Unicode::compose(str)
+    Compose Unicode string. Before composing, the trailing
+    characters are sorted in canonical order.
+    The parameter must be decomposed.
+    The composition is based on the reverse of the
+    character decomposition mapping in UnicodeData.txt,
+    CompositionExclusions.txt and the Hangul composition
+    algorithm.
+
+  Unicode::normalize_D(str)
+  Unicode::normalize_KD(str)
+    Normalizes Unicode string in form D or form KD.
+    These are aliases of decompose/decompose_compat.
+
+  Unicode::normalize_C(str)
+  Unicode::normalize_KC(str)
+    Normalizes Unicode string in form C or form KC.
+      normalize_C  = decompose + compose
+      normalize_KC = decompose_compat + compose
+
+  Unicode::upcase(str)
+  Unicode::downcase(str)
+  Unicode::capitalize(str)
+    Case conversion functions.
+    The mappings which these functions use are not normative
+    in UnicodeData.txt.
+
+- Bugs
+
+  UTR #15 suggests that the look up for Normalization Form C
+  should not be implemented with a hash of string for better
+  performance.
+
+
+- Copying
+
+  This extension module is copyrighted free software by
+  Yoshida Masato.
+
+  You can redistribute it and/or modify it under the same
+  term as Ruby.
+
+
+- Author
+
+  Yoshida Masato <yoshidam@yoshidam.net>
+
+
+- History
+
+  Nov 23, 1999 version 0.1

data/bin/mkunidata.rb

@@ -0,0 +1,128 @@
+#! /usr/local/bin/ruby -KU
+
+if $KCODE != 'UTF8'
+  raise "$KCODE must be UTF8"
+end
+
+HEAD=<<EOS
+/*
+ * UnicodeData
+ * 1999 by yoshidam
+ *
+ */
+
+#ifndef _UNIDATA_MAP
+#define _UNIDATA_MAP
+
+struct unicode_data {
+  const int code;
+  const int combining_class;
+  const int exclusion;
+  const char* const canon;
+  const char* const compat;
+  const int uppercase;
+  const int lowercase;
+  const int titlecase;
+};
+
+const static struct unicode_data unidata[] = {
+EOS
+
+TAIL=<<EOS
+};
+
+#endif
+EOS
+
+def hex2str(hex)
+  if hex.nil? || hex == ''
+    return [nil, nil]
+  end
+  canon = ""
+  compat = ""
+  chars = hex.split(" ")
+  if chars[0] =~ /^[0-9A-F]{4}$/
+    chars.each do |c|
+      canon << [c.hex].pack("U")
+    end
+    compat = canon
+  elsif chars[0] =~ /^<.+>$/
+    chars.shift
+    chars.each do |c|
+      compat << [c.hex].pack("U")
+    end
+    canon = nil
+  else
+    raise "unknown value: " + hex
+  end
+  [canon, compat]
+end
+
+def hex_or_nil(str)
+  return "-1" if str.nil?
+  return format("0x%04x", str.hex)
+end
+
+def printstr(str)
+  return "NULL" if !str
+  ret = ""
+  str.each_byte do |c|
+    if c >= 32 && c < 127 && c != 34 && c != 92
+      ret << c
+    else
+      ret << format("\\%03o", c)
+    end
+  end
+  return '"' + ret + '"'
+end
+
+## scan Composition Exclusions
+exclusion = {}
+open(ARGV[1]) do |f|
+  while l = f.gets
+    next if l =~ /^\#/ || l =~ /^$/
+    code, = l.split(/\s/)
+    code = code.hex
+    exclusion[code] = true
+  end
+end
+
+## scan UnicodeData
+udata = {}
+open(ARGV[0]) do |f|
+  while l = f.gets
+    l.chomp!
+    code, charname, gencat, ccclass, bidicat,decomp,
+      dec, digit, num, mirror, uni1_0, comment, upcase,
+      lowcase, titlecase = l.split(";");
+    code = code.hex
+    ccclass = ccclass.to_i
+    canon, compat = hex2str(decomp)
+    upcase = hex_or_nil(upcase)
+    lowcase = hex_or_nil(lowcase)
+    titlecase = hex_or_nil(titlecase)
+    udata[code] = [ccclass, canon, compat, upcase, lowcase, titlecase]
+  end
+end
+
+print HEAD
+udata.sort.each do |code, data|
+  ccclass, canon, compat, upcase, lowcase, titlecase = data
+  ## Exclusions
+  ex = 0
+  if exclusion[code]  ## Script-specifics or Post Composition Version
+    ex = 1
+  elsif canon =~ /^.$/ ## Singltons
+    ex = 2
+  elsif !canon.nil?
+    starter = canon.unpack("U*")[0]
+    if udata[starter][0] != 0 ## Non-stater decompositions
+      ex = 3
+    end
+  end
+  printf("  { 0x%04x, %d, %d, %s, %s, %s, %s, %s }, \n",
+         code, ccclass, ex, printstr(canon),
+         printstr(compat), upcase, lowcase, titlecase)
+end
+printf("  { -1, 0, 0, NULL, NULL, -1, -1, -1 }\n")
+print TAIL

data/ext/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile("unicode")

data/ext/unicode.c

@@ -0,0 +1,666 @@
+/*
+ * Unicode Library version 0.1
+ * Nov 23, 1999 yoshidam
+ *
+ */
+
+#include "ruby.h"
+#include "rubyio.h"
+#include <stdio.h>
+#include "wstring.h"
+#include "unidata.map"
+
+static VALUE mUnicode;
+static VALUE unicode_data;
+static VALUE composition_table;
+
+/* Hangul */
+#define SBASE   (0xac00)
+#define LBASE   (0x1100)
+#define LCOUNT  (19)
+#define VBASE   (0x1161)
+#define VCOUNT  (21)
+#define TBASE   (0x11a7)
+#define TCOUNT  (28)
+#define NCOUNT  (VCOUNT * TCOUNT) /* 588 */
+#define SCOUNT  (LCOUNT * NCOUNT) /* 11172 */
+
+static int
+get_cc(int ucs)
+{
+  VALUE ch = rb_hash_aref(unicode_data, INT2FIX(ucs));
+
+  if (!NIL_P(ch)) {
+    return unidata[FIX2INT(ch)].combining_class;
+  }
+  return 0;
+}
+
+static const char*
+get_canon(int ucs)
+{
+  VALUE ch = rb_hash_aref(unicode_data, INT2FIX(ucs));
+
+  if (!NIL_P(ch)) {
+    return unidata[FIX2INT(ch)].canon;
+  }
+  return NULL;
+}
+
+static const char*
+get_compat(int ucs)
+{
+  VALUE ch = rb_hash_aref(unicode_data, INT2FIX(ucs));
+
+  if (!NIL_P(ch)) {
+    return unidata[FIX2INT(ch)].compat;
+  }
+  return NULL;
+}
+
+static const int
+get_uppercase(int ucs)
+{
+  VALUE ch = rb_hash_aref(unicode_data, INT2FIX(ucs));
+
+  if (!NIL_P(ch)) {
+    int uc = unidata[FIX2INT(ch)].uppercase;
+    if (uc > 0) return uc;
+  }
+  return ucs;
+}
+
+static int
+get_lowercase(int ucs)
+{
+  VALUE ch = rb_hash_aref(unicode_data, INT2FIX(ucs));
+
+  if (!NIL_P(ch)) {
+    int lc = unidata[FIX2INT(ch)].lowercase;
+    if (lc > 0) return lc;
+  }
+  return ucs;
+}
+
+static int
+get_titlecase(int ucs)
+{
+  VALUE ch = rb_hash_aref(unicode_data, INT2FIX(ucs));
+
+  if (!NIL_P(ch)) {
+    int tc = unidata[FIX2INT(ch)].titlecase;
+    if (tc > 0) return tc;
+  }
+  return ucs;
+}
+
+static int
+get_composition(const char* str)
+{
+  VALUE ch = rb_hash_aref(composition_table, rb_str_new2(str));
+
+  if (!NIL_P(ch)) {
+    return FIX2INT(ch);
+  }
+  return -1;
+}
+
+static WString*
+sort_canonical(WString* ustr)
+{
+  int i = 1;
+  int len = ustr->len;
+
+  if (len < 2) return ustr;
+
+  while (i < len) {
+    int last = ustr->str[i - 1];
+    int ch = ustr->str[i];
+    int last_cc = get_cc(last);
+    int cc = get_cc(ch);
+    if (cc != 0 && last_cc != 0 && last_cc > cc) {
+      ustr->str[i] = last;
+      ustr->str[i-1] = ch;
+      if (i > 1) i--;
+    }
+    else {
+      i++;
+    }
+  }
+  return ustr;
+}
+
+static void
+decompose_hangul(int ucs, int* l, int* v, int* t)
+{
+  int sindex = ucs - SBASE;
+  if (sindex < 0 || sindex >= SCOUNT) {
+    *l = ucs;
+    *v = *t = 0;
+    return;
+  }
+  *l = LBASE + sindex / NCOUNT;
+  *v = VBASE + (sindex % NCOUNT) / TCOUNT;
+  *t = TBASE + sindex % TCOUNT;
+  if (*t == TBASE) *t = 0;
+}
+
+/*
+ * push decomposed str into result 
+ */
+static WString*
+decompose_internal(WString* ustr, WString* result)
+{
+  int i;
+  int len = ustr->len;
+
+  for (i = 0; i < len; i++) {
+    int ucs = ustr->str[i];
+    if (ucs >= SBASE && ucs < SBASE + SCOUNT) {
+      int l, v, t;
+      decompose_hangul(ucs, &l, &v, &t);
+      WStr_addWChar(result, l);
+      if (v) WStr_addWChar(result, v);
+      if (t) WStr_addWChar(result, t);
+    }
+    else {
+      const char* dc = get_canon(ucs);
+      if (!dc) {
+	WStr_addWChar(result, ucs);
+      }
+      else {
+	WString wdc;
+	WStr_allocWithUTF8(&wdc, dc);
+	decompose_internal(&wdc, result);
+	WStr_free(&wdc);
+      }
+    }
+  }
+  return result;
+}
+
+/*
+ * push compatibility decomposed str into result 
+ */
+static WString*
+decompose_compat_internal(WString* ustr, WString* result)
+{
+  int i;
+  int len = ustr->len;
+
+  for (i = 0; i < len; i++) {
+    int ucs = ustr->str[i];
+    if (ucs >= SBASE && ucs < SBASE + SCOUNT) {
+      int l, v, t;
+      decompose_hangul(ucs, &l, &v, &t);
+      WStr_addWChar(result, l);
+      if (v) WStr_addWChar(result, v);
+      if (t) WStr_addWChar(result, t);
+    }
+    else {
+      const char* dc = get_compat(ucs);
+      if (!dc) {
+	WStr_addWChar(result, ucs);
+      }
+      else {
+	WString wdc;
+	WStr_allocWithUTF8(&wdc, dc);
+	decompose_compat_internal(&wdc, result);
+	WStr_free(&wdc);
+      }
+    }
+  }
+  return result;
+}
+
+
+#define UCS4toUTF8(p, c) \
+  do { \
+    if (c < 128) { \
+      *p++ = c; \
+    } \
+    else if (c < 2048) { \
+      *p++ = (c >> 6) | 192; \
+      *p++ = (c & 63) | 128; \
+    } \
+    else if (c < 0x10000) { \
+      *p++ = (c >> 12) | 224; \
+      *p++ = ((c >> 6) & 63) | 128; \
+      *p++ = (c & 63) | 128; \
+    } \
+    else if (c < 0x200000) { \
+      *p++ = (c >> 18) | 240; \
+      *p++ = ((c >> 12) & 63) | 128; \
+      *p++ = ((c >> 6) & 63) | 128; \
+      *p++ = (c & 63) | 128; \
+    } \
+    else if (c < 0x4000000) { \
+      *p++ = (c >> 24) | 248; \
+      *p++ = ((c >> 18) & 63) | 128; \
+      *p++ = ((c >> 12) & 63) | 128; \
+      *p++ = ((c >> 6) & 63) | 128; \
+      *p++ = (c & 63) | 128; \
+    } \
+    else if (c < 0x80000000) { \
+      *p++ = (c >> 30) | 252; \
+      *p++ = ((c >> 24) & 63) | 128; \
+      *p++ = ((c >> 18) & 63) | 128; \
+      *p++ = ((c >> 12) & 63) | 128; \
+      *p++ = ((c >> 6) & 63) | 128; \
+      *p++ = (c & 63) | 128; \
+    } \
+  } while (0)
+
+static int
+compose_pair(int c1, int c2)
+{
+  int ret;
+  char ustr[13]; /* stored two UTF-8 chars */
+  char *p = ustr;
+
+  /* Hangul L + V */
+  if (c1 >= LBASE && c1 < LBASE + LCOUNT &&
+      c2 >= VBASE && c2 < VBASE + VCOUNT) {
+    return SBASE + ((c1 - LBASE) * VCOUNT + (c2 - VBASE)) * TCOUNT;
+  }
+  /* Hangul LV + T */
+  else if (c1 >= SBASE && c1 < SBASE + SCOUNT &&
+	   (c1 - SBASE) % TCOUNT == 0 &&
+	   c2 >= TBASE && c2 < TBASE + TCOUNT) {
+    return c1 + (c2 - TBASE);
+  }
+  UCS4toUTF8(p, c1);
+  UCS4toUTF8(p, c2);
+  *p = '\0';
+  ret = get_composition(ustr);
+
+  return ret;
+}
+
+/*
+ * push canonical composed str into result 
+ */
+static WString*
+compose_internal(WString* ustr, WString* result)
+{
+  int len = ustr->len;
+  int starter;
+  int startercc;
+  int i;
+
+  if (len == 0) return result;
+
+  starter = ustr->str[0];
+  startercc = get_cc(starter);
+  if (startercc != 0) startercc = 256;
+  for (i = 1; i < len; i++) {
+    int ch = ustr->str[i];
+    int cc = get_cc(ch);
+    int composite;
+
+    if (startercc == 0 &&
+	(composite = compose_pair(starter, ch)) >= 0) {
+      starter = composite;
+      startercc = get_cc(composite);
+    }
+    else {
+      WStr_addWChar(result, starter);
+      starter = ch;
+      startercc = cc;
+    }
+  }
+  WStr_addWChar(result, starter);
+
+  return result;
+}
+
+static WString*
+upcase_internal(WString* str)
+{
+  int i;
+
+  for (i = 0; i < str->len; i++) {
+    int uc = get_uppercase(str->str[i]);
+    if (uc > 0) str->str[i] = uc;
+  }
+
+  return str;
+}
+
+static WString*
+downcase_internal(WString* str)
+{
+  int i;
+
+  for (i = 0; i < str->len; i++) {
+    int lc = get_lowercase(str->str[i]);
+    if (lc > 0) str->str[i] = lc;
+  }
+
+  return str;
+}
+
+static WString*
+capitalize_internal(WString* str)
+{
+  int i;
+
+  if (str->len > 1) {
+    int tc = get_titlecase(str->str[0]);
+    if (tc > 0) str->str[0] = tc;
+  }
+  for (i = 1; i < str->len; i++) {
+    int lc = get_lowercase(str->str[i]);
+    if (lc > 0) str->str[i] = lc;
+  }
+
+  return str;
+}
+
+static VALUE
+unicode_strcmp(VALUE obj, VALUE str1, VALUE str2)
+{
+  WString wstr1;
+  WString wstr2;
+  WString result1;
+  WString result2;
+  UString ustr1;
+  UString ustr2;
+  int ret;
+
+  Check_Type(str1, T_STRING);
+  Check_Type(str2, T_STRING);
+  WStr_allocWithUTF8(&wstr1, RSTRING(str1)->ptr);
+  WStr_allocWithUTF8(&wstr2, RSTRING(str2)->ptr);
+  WStr_alloc(&result1);
+  WStr_alloc(&result2);
+  decompose_internal(&wstr1, &result1);
+  decompose_internal(&wstr2, &result2);
+  WStr_free(&wstr1);
+  WStr_free(&wstr2);
+  sort_canonical(&result1);
+  sort_canonical(&result2);
+  UStr_alloc(&ustr1);
+  UStr_alloc(&ustr2);
+  WStr_convertIntoUString(&result1, &ustr1);
+  WStr_convertIntoUString(&result2, &ustr2);
+  WStr_free(&result1);
+  WStr_free(&result2);
+  UStr_addChar(&ustr1, '\0');
+  UStr_addChar(&ustr2, '\0');
+  ret = strcmp(ustr1.str, ustr2.str);
+  UStr_free(&ustr1);
+  UStr_free(&ustr2);
+
+  return INT2FIX(ret);
+}
+
+static VALUE
+unicode_strcmp_compat(VALUE obj, VALUE str1, VALUE str2)
+{
+  WString wstr1;
+  WString wstr2;
+  WString result1;
+  WString result2;
+  UString ustr1;
+  UString ustr2;
+  int ret;
+
+  Check_Type(str1, T_STRING);
+  Check_Type(str2, T_STRING);
+  WStr_allocWithUTF8(&wstr1, RSTRING(str1)->ptr);
+  WStr_allocWithUTF8(&wstr2, RSTRING(str2)->ptr);
+  WStr_alloc(&result1);
+  WStr_alloc(&result2);
+  decompose_compat_internal(&wstr1, &result1);
+  decompose_compat_internal(&wstr2, &result2);
+  WStr_free(&wstr1);
+  WStr_free(&wstr2);
+  sort_canonical(&result1);
+  sort_canonical(&result2);
+  UStr_alloc(&ustr1);
+  UStr_alloc(&ustr2);
+  WStr_convertIntoUString(&result1, &ustr1);
+  WStr_convertIntoUString(&result2, &ustr2);
+  WStr_free(&result1);
+  WStr_free(&result2);
+  UStr_addChar(&ustr1, '\0');
+  UStr_addChar(&ustr2, '\0');
+  ret = strcmp(ustr1.str, ustr2.str);
+  UStr_free(&ustr1);
+  UStr_free(&ustr2);
+
+  return INT2FIX(ret);
+}
+
+static VALUE
+unicode_decompose(VALUE obj, VALUE str)
+{
+  WString ustr;
+  WString result;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr, RSTRING(str)->ptr);
+  WStr_alloc(&result);
+  decompose_internal(&ustr, &result);
+  WStr_free(&ustr);
+  sort_canonical(&result);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&result, &ret);
+  WStr_free(&result);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_decompose_compat(VALUE obj, VALUE str)
+{
+  WString ustr;
+  WString result;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr, RSTRING(str)->ptr);
+  WStr_alloc(&result);
+  decompose_compat_internal(&ustr, &result);
+  WStr_free(&ustr);
+  sort_canonical(&result);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&result, &ret);
+  WStr_free(&result);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_compose(VALUE obj, VALUE str)
+{
+  WString ustr;
+  WString result;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr, RSTRING(str)->ptr);
+  sort_canonical(&ustr);
+  WStr_alloc(&result);
+  compose_internal(&ustr, &result);
+  WStr_free(&ustr);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&result, &ret);
+  WStr_free(&result);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_normalize_C(VALUE obj, VALUE str)
+{
+  WString ustr1;
+  WString ustr2;
+  WString result;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr1, RSTRING(str)->ptr);
+  WStr_alloc(&ustr2);
+  decompose_internal(&ustr1, &ustr2);
+  WStr_free(&ustr1);
+  sort_canonical(&ustr2);
+  WStr_alloc(&result);
+  compose_internal(&ustr2, &result);
+  WStr_free(&ustr2);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&result, &ret);
+  WStr_free(&result);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_normalize_KC(VALUE obj, VALUE str)
+{
+  WString ustr1;
+  WString ustr2;
+  WString result;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr1, RSTRING(str)->ptr);
+  WStr_alloc(&ustr2);
+  decompose_compat_internal(&ustr1, &ustr2);
+  WStr_free(&ustr1);
+  sort_canonical(&ustr2);
+  WStr_alloc(&result);
+  compose_internal(&ustr2, &result);
+  WStr_free(&ustr2);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&result, &ret);
+  WStr_free(&result);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_upcase(VALUE obj, VALUE str)
+{
+  WString ustr;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr, RSTRING(str)->ptr);
+  upcase_internal(&ustr);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&ustr, &ret);
+  WStr_free(&ustr);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_downcase(VALUE obj, VALUE str)
+{
+  WString ustr;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr, RSTRING(str)->ptr);
+  downcase_internal(&ustr);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&ustr, &ret);
+  WStr_free(&ustr);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+static VALUE
+unicode_capitalize(VALUE obj, VALUE str)
+{
+  WString ustr;
+  UString ret;
+  VALUE vret;
+
+  Check_Type(str, T_STRING);
+  WStr_allocWithUTF8(&ustr, RSTRING(str)->ptr);
+  capitalize_internal(&ustr);
+  UStr_alloc(&ret);
+  WStr_convertIntoUString(&ustr, &ret);
+  WStr_free(&ustr);
+  vret = rb_str_new(ret.str, ret.len);
+  UStr_free(&ret);
+
+  return vret;
+}
+
+void
+Init_unicode()
+{
+  int i;
+
+  mUnicode = rb_define_module("Unicode");
+  unicode_data = rb_hash_new();
+  composition_table = rb_hash_new();
+
+  rb_global_variable(&unicode_data);
+  rb_global_variable(&composition_table);
+
+  for (i = 0; unidata[i].code != -1; i++) {
+    int code = unidata[i].code;
+    const char* canon = unidata[i].canon;
+    int exclusion = unidata[i].exclusion;
+
+    rb_hash_aset(unicode_data, INT2FIX(code), INT2FIX(i));
+    if (canon && exclusion == 0) {
+      rb_hash_aset(composition_table, rb_str_new2(canon), INT2FIX(code));
+    }
+  }
+
+  rb_define_module_function(mUnicode, "strcmp",
+			    unicode_strcmp, 2);
+  rb_define_module_function(mUnicode, "strcmp_compat",
+			    unicode_strcmp_compat, 2);
+
+  rb_define_module_function(mUnicode, "decompose",
+			    unicode_decompose, 1);
+  rb_define_module_function(mUnicode, "decompose_compat",
+			    unicode_decompose_compat, 1);
+  rb_define_module_function(mUnicode, "compose",
+			    unicode_compose, 1);
+
+  rb_define_module_function(mUnicode, "normalize_D",
+			    unicode_decompose, 1);
+  rb_define_module_function(mUnicode, "normalize_KD",
+			    unicode_decompose_compat, 1);
+  rb_define_module_function(mUnicode, "normalize_C",
+			    unicode_normalize_C, 1);
+  rb_define_module_function(mUnicode, "normalize_KC",
+			    unicode_normalize_KC, 1);
+
+  rb_define_module_function(mUnicode, "upcase",
+			    unicode_upcase, 1);
+  rb_define_module_function(mUnicode, "downcase",
+			    unicode_downcase, 1);
+  rb_define_module_function(mUnicode, "capitalize",
+			    unicode_capitalize, 1);
+}