ffi-icu 0.0.1 → 0.0.2

data/.gitignore

@@ -19,3 +19,4 @@ rdoc
 pkg
 
 ## PROJECT::SPECIFIC
+*.rbc

data/README.rdoc

@@ -1,10 +1,19 @@
 = ffi-icu
 
-Simple FFI wrappers for things I need from ICU.
+Simple FFI wrappers for things I need from ICU. For the full thing, check out ICU4R instead.
+
+= Gem
+
+* http://rubygems.org/gems/ffi-icu
+
+  gem install ffi-icu
 
 = Dependencies
 
-ICU - you might need to hack the ffi_lib call to make it work. Please send a patch if you do!
+ICU. If you get messages that the library or functions are not found, you can
+set some environment varibles to tell ffi-icu where to find it, i.e.:
+
+  FFI_ICU_LIB="icui18n.so" FFI_ICU_VERSION_SUFFIX="_3_8" ruby -r ffi-icu
 
 = Features
 
@@ -54,7 +63,12 @@ Rubies:
 * MRI 1.9.1
 * MRI 1.8.7
 
-YMMV.
+= TODO:
+
+* Useful ICU stuff:
+  - number formatting (decimal points, thousand separators, currency)
+  - date formatting
+* Windows?!
 
 == Note on Patches/Pull Requests
 

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/ffi-icu.rb

@@ -14,10 +14,20 @@ module ICU
       os
     end
   end
+
+  def self.ruby19?
+    RUBY_VERSION >= '1.9'
+  end
 end
 
 require "ffi-icu/lib"
 require "ffi-icu/uchar"
 require "ffi-icu/chardet"
 require "ffi-icu/collation"
+require "ffi-icu/transliteration"
+require "ffi-icu/normalization"
 
+unless ICU.ruby19?
+  require 'jcode'
+  $KCODE = 'u'
+end

data/lib/ffi-icu/chardet.rb

@@ -44,7 +44,7 @@ module ICU
       def detect_all(str)
         set_text(str)
 
-        matches_found_ptr = FFI::MemoryPointer.new :int32
+        matches_found_ptr = FFI::MemoryPointer.new :int32_t
         array_ptr = Lib.check_error do |status|
           Lib.ucsdet_detectAll(@detector, matches_found_ptr, status)
         end

data/lib/ffi-icu/collation.rb

@@ -59,8 +59,15 @@ module ICU
                                     UCharPointer.from_string(b), b.length)
       end
 
-      # can't override Object#equal? - suggestions welcome
-      def same?(a, b)
+      def equal?(*args)
+        return super() if args.empty?
+
+        if args.size != 2
+          raise ArgumentError, "wrong number of arguments (#{args.size} for 2)"
+        end
+
+        a, b = args
+
         Lib.ucol_equal(@c, UCharPointer.from_string(a), a.length,
                            UCharPointer.from_string(b), b.length)
       end

data/lib/ffi-icu/lib.rb

@@ -2,6 +2,9 @@ module ICU
   class Error < StandardError
   end
 
+  class BufferOverflowError < StandardError
+  end
+
   module Lib
     extend FFI::Library
 
@@ -14,6 +17,21 @@ module ICU
     def self.find_icu
       suffix = ''
 
+      # let the user tell us where the lib is
+      if ENV['FFI_ICU_LIB']
+        libs = ENV['FFI_ICU_LIB'].split(",")
+        ffi_lib *libs
+
+        if ENV['FFI_ICU_VERSION_SUFFIX']
+          return ENV['FFI_ICU_VERSION_SUFFIX']
+        elsif num = libs.first[/\d+$/]
+          return num.split(//).join("_")
+        else
+          return suffix
+        end
+      end
+
+      # ok, try to find it
       case ICU.platform
       when :osx
         ffi_lib "icucore"
@@ -40,7 +58,12 @@ module ICU
       error_code = ptr.read_int
 
       if error_code > 0
-        raise Error, "#{Lib.u_errorName error_code}"
+        name = Lib.u_errorName error_code
+        if name == "U_BUFFER_OVERFLOW_ERROR"
+          raise BufferOverflowError
+        else
+          raise Error, name
+        end
       elsif error_code < 0
         warn "ffi-icu: #{Lib.u_errorName error_code}"
       end
@@ -49,12 +72,14 @@ module ICU
     end
 
     def self.enum_ptr_to_array(enum_ptr)
-      length = Lib.check_error do |status|
-        Lib.uenum_count(enum_ptr, status)
+      length = check_error do |status|
+        uenum_count(enum_ptr, status)
       end
 
+      len = FFI::MemoryPointer.new(:int)
+
       (0...length).map do |idx|
-        Lib.check_error { |st| Lib.uenum_next(enum_ptr, nil, st) }
+        check_error { |st| uenum_next(enum_ptr, len, st) }
       end
     end
 
@@ -71,7 +96,8 @@ module ICU
     attach_function :uenum_count, "uenum_count#{suffix}", [:pointer, :pointer], :int
     attach_function :uenum_close, "uenum_close#{suffix}",  [:pointer], :void
     attach_function :uenum_next, "uenum_next#{suffix}",  [:pointer, :pointer, :pointer], :string
-
+    attach_function :u_charsToUChars, "u_charsToUChars#{suffix}", [:string, :pointer, :int32_t], :void
+    attach_function :u_UCharsToChars, "u_UCharsToChars#{suffix}", [:pointer, :string, :int32_t], :void
 
     # CharDet
     #
@@ -80,12 +106,12 @@ module ICU
 
     attach_function :ucsdet_open, "ucsdet_open#{suffix}",  [:pointer], :pointer
     attach_function :ucsdet_close, "ucsdet_close#{suffix}",  [:pointer], :void
-    attach_function :ucsdet_setText, "ucsdet_setText#{suffix}",  [:pointer, :string, :int32, :pointer], :void
-    attach_function :ucsdet_setDeclaredEncoding, "ucsdet_setDeclaredEncoding#{suffix}",  [:pointer, :string, :int32, :pointer], :void
+    attach_function :ucsdet_setText, "ucsdet_setText#{suffix}",  [:pointer, :string, :int32_t, :pointer], :void
+    attach_function :ucsdet_setDeclaredEncoding, "ucsdet_setDeclaredEncoding#{suffix}",  [:pointer, :string, :int32_t, :pointer], :void
     attach_function :ucsdet_detect, "ucsdet_detect#{suffix}",  [:pointer, :pointer], :pointer
     attach_function :ucsdet_detectAll, "ucsdet_detectAll#{suffix}",  [:pointer, :pointer, :pointer], :pointer
     attach_function :ucsdet_getName, "ucsdet_getName#{suffix}",  [:pointer, :pointer], :string
-    attach_function :ucsdet_getConfidence, "ucsdet_getConfidence#{suffix}",  [:pointer, :pointer], :int32
+    attach_function :ucsdet_getConfidence, "ucsdet_getConfidence#{suffix}",  [:pointer, :pointer], :int32_t
     attach_function :ucsdet_getLanguage, "ucsdet_getLanguage#{suffix}",  [:pointer, :pointer], :string
     attach_function :ucsdet_getAllDetectableCharsets, "ucsdet_getAllDetectableCharsets#{suffix}",  [:pointer, :pointer], :pointer
     attach_function :ucsdet_isInputFilterEnabled, "ucsdet_isInputFilterEnabled#{suffix}",  [:pointer], :bool
@@ -98,15 +124,60 @@ module ICU
 
     attach_function :ucol_open, "ucol_open#{suffix}",  [:string, :pointer], :pointer
     attach_function :ucol_close, "ucol_close#{suffix}",  [:pointer], :void
-    attach_function :ucol_strcoll, "ucol_strcoll#{suffix}",  [:pointer, :pointer, :int32, :pointer, :int32], :int
+    attach_function :ucol_strcoll, "ucol_strcoll#{suffix}",  [:pointer, :pointer, :int32_t, :pointer, :int32_t], :int
     attach_function :ucol_getKeywords, "ucol_getKeywords#{suffix}",  [:pointer], :pointer
     attach_function :ucol_getKeywordValues, "ucol_getKeywordValues#{suffix}",  [:string, :pointer], :pointer
-    attach_function :ucol_getAvailable, "ucol_getAvailable#{suffix}", [:int32], :string
-    attach_function :ucol_countAvailable, "ucol_countAvailable#{suffix}", [], :int32
+    attach_function :ucol_getAvailable, "ucol_getAvailable#{suffix}", [:int32_t], :string
+    attach_function :ucol_countAvailable, "ucol_countAvailable#{suffix}", [], :int32_t
     attach_function :ucol_getLocale, "ucol_getLocale#{suffix}", [:pointer, :int, :pointer], :string
-    attach_function :ucol_greater, "ucol_greater#{suffix}", [:pointer, :pointer, :int32, :pointer, :int32], :bool
-    attach_function :ucol_greaterOrEqual, "ucol_greaterOrEqual#{suffix}", [:pointer, :pointer, :int32, :pointer, :int32], :bool
-    attach_function :ucol_equal, "ucol_equal#{suffix}", [:pointer, :pointer, :int32, :pointer, :int32], :bool
+    attach_function :ucol_greater, "ucol_greater#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t], :bool
+    attach_function :ucol_greaterOrEqual, "ucol_greaterOrEqual#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t], :bool
+    attach_function :ucol_equal, "ucol_equal#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t], :bool
+
+    # Transliteration
+    #
+    # http://icu-project.org/apiref/icu4c/utrans_8h.html
+    #
+
+    class UParseError < FFI::Struct
+      layout :line,         :int32_t,
+             :offset,       :int32_t,
+             :pre_context,  :pointer,
+             :post_context, :pointer
+
+
+    end
+
+    class UTransPosition < FFI::Struct
+      layout :context_start, :int32_t,
+             :context_limit, :int32_t,
+             :start,         :int32_t,
+             :end,           :int32_t
+
+    end
+
+    enum :trans_direction, [:forward, :reverse]
+
+    attach_function :utrans_openIDs, "utrans_openIDs#{suffix}", [:pointer], :pointer
+    attach_function :utrans_openU, "utrans_openU#{suffix}", [:pointer, :int32_t, :trans_direction, :pointer, :int32_t, :pointer, :pointer], :pointer
+    attach_function :utrans_open, "utrans_open#{suffix}", [:string, :trans_direction, :pointer, :int32_t, :pointer, :pointer], :pointer
+    attach_function :utrans_close, "utrans_close#{suffix}", [:pointer], :void
+    attach_function :utrans_transUChars, "utrans_transUChars#{suffix}", [:pointer, :pointer, :pointer, :int32_t, :int32_t, :pointer, :pointer], :void
+
+    # Normalization
+    #
+    # http://icu-project.org/apiref/icu4c/unorm_8h.html
+    #
+
+    enum :normalization_mode, [ :none,    1,
+                                :nfd,     2,
+                                :nfkd,    3,
+                                :nfc,     4,
+                                :default, 4,
+                                :nfkc,    5,
+                                :fcd,     6
+                              ]
 
+    attach_function :unorm_normalize, "unorm_normalize#{suffix}", [:pointer, :int32_t, :normalization_mode, :int32_t, :pointer, :int32_t, :pointer], :int32_t
   end # Lib
 end # ICU

data/lib/ffi-icu/normalization.rb

@@ -0,0 +1,29 @@
+module ICU
+  module Normalization
+
+    def self.normalize(input, mode = :default)
+      input_length = ICU.ruby19? ? input.length : input.jlength
+      needed_length  = 0
+      result_length = 0
+
+      retried = false
+      ptr     = nil
+
+      begin
+        Lib.check_error do |error|
+          needed_length = Lib.unorm_normalize(UCharPointer.from_string(input), input_length, mode, 0, ptr, result_length, error)
+        end
+      rescue BufferOverflowError
+        raise if retried
+        ptr     = UCharPointer.from_string("\0" * needed_length)
+        result_length = needed_length + 1
+
+        retried = true
+        retry
+      end
+
+      ptr.string if ptr
+    end
+
+  end # Normalization
+end # ICU

data/lib/ffi-icu/transliteration.rb

@@ -0,0 +1,79 @@
+module ICU
+  module Transliteration
+
+    class << self
+      def transliterate(translit_id, str)
+        t = Transliterator.new translit_id
+        res = t.transliterate str
+        t.close
+
+        res
+      end
+      alias_method :translit, :transliterate
+
+      def available_ids
+        enum_ptr = Lib.check_error do |error|
+          Lib.utrans_openIDs(error)
+        end
+
+        result = Lib.enum_ptr_to_array(enum_ptr)
+        Lib.uenum_close(enum_ptr)
+
+        result
+      end
+    end
+
+    class Transliterator
+
+      def initialize(id, direction = :forward)
+        @parse_error = Lib::UParseError.new
+        Lib.check_error do |status|
+          # couldn't get utrans_openU to work properly, so using deprecated utrans_open for now
+          @tr = Lib.utrans_open(id, direction, nil, 0, @parse_error, status)
+        end
+      end
+
+      def transliterate(from)
+        # this is a bit unpleasant
+
+        unicode_size = from.unpack("U*").size
+        capacity     = from.bytesize + 1
+        buf          = UCharPointer.from_string(from)
+        limit        = FFI::MemoryPointer.new :int32
+        text_length  = FFI::MemoryPointer.new :int32
+
+        retried = false
+
+        begin
+          # resets to original size on retry
+          [limit, text_length].each do |ptr|
+            ptr.put_int32(0, unicode_size)
+          end
+
+          Lib.check_error do |error|
+            Lib.utrans_transUChars(@tr, buf, text_length, capacity, 0, limit, error)
+          end
+        rescue BufferOverflowError
+          new_size = text_length.get_int32(0)
+          $stderr.puts "BufferOverflowError, needs: #{new_size}" if $DEBUG
+
+          raise BufferOverflowError, "needed #{new_size}" if retried
+
+          capacity = new_size + 1
+          buf      = buf.resized_to capacity
+          retried  = true
+
+          retry
+        end
+
+        buf.string text_length.get_int32(0)
+      end
+
+      def close
+        Lib.utrans_close @tr
+      end
+
+    end # Transliterator
+  end # Translit
+end # ICU
+

data/lib/ffi-icu/uchar.rb

@@ -1,11 +1,34 @@
 module ICU
-
   class UCharPointer < FFI::MemoryPointer
+
+    UCHAR_TYPE = :uint16 # not sure how platform-dependent this is..
+    TYPE_SIZE  = FFI.type_size(UCHAR_TYPE)
+
     def self.from_string(str)
-      # not sure how this will work with other encodings
-      str = str.encode("UTF-8") if str.respond_to? :encode
-      super str.unpack("U*").pack("L*")
+      str   = str.encode("UTF-8") if str.respond_to? :encode
+      bytes = str.unpack("U*")
+
+      ptr = new UCHAR_TYPE, bytes.size
+      ptr.put_array_of_uint16 0, bytes
+
+      ptr
+    end
+
+    def resized_to(new_size)
+      raise "new_size must be larger than current size" if new_size < size
+      resized = self.class.new UCHAR_TYPE, new_size
+      resized.put_bytes(0, get_bytes(0, size))
+
+      resized
+    end
+
+    def string(length = nil)
+      length ||= size / TYPE_SIZE
+
+      wstring = get_array_of_uint16(0, length)
+      wstring.pack("U*")
     end
-  end
 
-end
+
+  end # UCharPointer
+end # ICU

data/spec/collation_spec.rb

@@ -6,7 +6,7 @@ module ICU
   module Collation
     describe Collator do
 
-      before { @c = Collator.new("no") }
+      before { @c = Collator.new("nb") }
       after { @c.close }
 
       it "should collate an array of strings" do
@@ -43,7 +43,8 @@ module ICU
       end
 
       it "should know if a string is equal to another" do
-        @c.should be_same("a", "a")
+        @c.should be_equal("a", "a")
+        @c.should_not be_equal("a", "b")
       end
 
     end

data/spec/normalization_spec.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+module ICU
+  module Normalization
+    #  http://bugs.icu-project.org/trac/browser/icu/trunk/source/test/cintltst/cnormtst.c
+
+    describe "Normalization" do
+
+      it "should normalize a string - decomposed" do
+        ICU::Normalization.normalize("Å", :nfd).unpack("U*").should == [65, 778]
+      end
+
+      it "should normalize a string - composed" do
+        ICU::Normalization.normalize("Å", :nfc).unpack("U*").should == [197]
+      end
+
+      # TODO: add more normalization tests
+
+
+    end
+  end # Normalization
+end # ICU

data/spec/transliteration_spec.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+require "spec_helper"
+
+module ICU
+  describe Transliteration::Transliterator do
+
+    def transliterator(*args)
+      @t = Transliteration::Transliterator.new(*args)
+    end
+
+    after { @t.close if @t }
+
+    [
+     { :id => "Any-Hex",     :input => "abcde",  :output => "\\u0061\\u0062\\u0063\\u0064\\u0065"  },
+     { :id => "Lower",       :input => "ABC",    :output => "abc"                                  },
+    ].each do |test|
+
+      it "should transliterate #{test[:id]}" do
+        transliterator(test[:id]).transliterate(test[:input]).should == test[:output]
+      end
+
+    end
+  end # Transliterator
+
+  describe Transliteration do
+    it "should provide a list of available ids" do
+      ids = ICU::Transliteration.available_ids
+      ids.should be_kind_of(Array)
+      ids.should_not be_empty
+    end
+  end # Transliteration
+end # ICU

data/test.c

@@ -0,0 +1,56 @@
+#include <unicode/unorm.h>
+#include <unicode/ustdio.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <iconv.h>
+#include <string.h>
+#include <stdlib.h>
+
+void print_uchars(UChar *str) {
+  UFILE *out = u_finit(stdout, NULL, NULL);
+  u_fprintf(out, "uchars: %S\n", str);
+  u_fclose(out);
+}
+
+void print_error(UErrorCode status) {
+  printf("err: %s (%d)\n", u_errorName(status), status);
+}
+
+int main (int argc, char const *argv[])
+{
+
+  UTransliterator* trans = NULL;
+  UErrorCode status = U_ZERO_ERROR;
+
+  trans = utrans_open("Any-Hex", UTRANS_FORWARD, NULL, 0, NULL, &status);
+  if(U_FAILURE(status)) {
+    print_error(status);
+    exit(1);
+  }
+
+  UChar from[256];
+  UChar buf[6];
+
+  int32_t text_length, limit;
+
+  u_uastrcpy(from, "abcde");
+  u_strcpy(buf, from);
+
+  limit = text_length = u_strlen(buf);
+  printf("limit: %d\n", limit);
+  printf("text_length: %d\n", limit);
+
+  utrans_transUChars(trans, buf, &text_length, 256, 0, &limit, &status);
+
+  printf("uchar ptr length after: %d\n", u_strlen(buf));
+  printf("text_length after: %d\n", text_length);
+
+  if(U_FAILURE(status)) {
+    print_error(status);
+    exit(1);
+  }
+
+  print_uchars(buf);
+
+  return 0;
+}

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ffi-icu
 version: !ruby/object:Gem::Version 
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Jari Bakken
@@ -14,16 +15,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-11 00:00:00 +02:00
+date: 2010-05-29 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: ffi
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 1
         segments: 
         - 0
         - 6
@@ -35,9 +38,11 @@ dependencies:
   name: rspec
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 27
         segments: 
         - 1
         - 3
@@ -67,11 +72,16 @@ files:
 - lib/ffi-icu/chardet.rb
 - lib/ffi-icu/collation.rb
 - lib/ffi-icu/lib.rb
+- lib/ffi-icu/normalization.rb
+- lib/ffi-icu/transliteration.rb
 - lib/ffi-icu/uchar.rb
 - spec/chardet_spec.rb
 - spec/collation_spec.rb
+- spec/normalization_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/transliteration_spec.rb
+- test.c
 has_rdoc: true
 homepage: http://github.com/jarib/ffi-icu
 licenses: []
@@ -82,27 +92,33 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Simple FFI wrappers for things I need from ICU.
 test_files: 
 - spec/chardet_spec.rb
+- spec/normalization_spec.rb
+- spec/transliteration_spec.rb
 - spec/spec_helper.rb
 - spec/collation_spec.rb