ffi-icu 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61818359548264beab4f803a6d4b6873eb9796f9
-  data.tar.gz: 5cffff3be406e40154bc45c51b2e199d3b36eb9c
+  metadata.gz: dcb52a7fa85970538582cc014850664a50a9725a
+  data.tar.gz: e226b65e6436f613207145582c1c6b0ccd9d2cf0
 SHA512:
-  metadata.gz: 4f212ef09319ea71ea83493578d659c8e44d8be59a5d9afeb72f14bfd08ca834e00026cf397eba9ebb8737011455bc4c717fd66b3684c00e6515409d80560ccb
-  data.tar.gz: 0ab9f362d3d94351fc720f95d90ab3718a299e0061a6fec80a2ae690d2a8d44556a8ef73d22d25b50f2d684c1a9dc2bd625030b9c05f2548406c48bde810b5ae
+  metadata.gz: 62af7b275d1e40bc2d39ce81792f6bff5dc38b4473fcbaf2c507ce8420dd93ce9a2ade21819281288cd893350e3afd930a065d7b6d9d0359e75fedad86a97325
+  data.tar.gz: b0e89ecc1ca69ddbdb4d6a4ae5818f763b69006609c642aecd484fed96c864bd0db3a0c48cba2e8e56bdae616dc1ff32e2995c126178334a0d6d9b923d09c8c6

data/README.md

@@ -127,19 +127,20 @@ Tested on:
 
 Platforms:
 
-* OS X 10.6
-* Arch Linux
+* OS X 10.6 - 10.10
+* Travis' Linux
 
 Rubies:
 
-* MRI 1.9.1
-* MRI 1.8.7
+* 1.9.3
+* 2.0.0
+* 2.1.0
+* ruby-head
 
 TODO:
 =====
 
-* Useful ICU stuff:
-  - date formatting
+* Any other useful part of ICU? 
 * Windows?!
 
 Note on Patches/Pull Requests

data/lib/ffi-icu.rb

@@ -38,7 +38,7 @@ require "ffi-icu/collation"
 require "ffi-icu/locale"
 require "ffi-icu/transliteration"
 require "ffi-icu/normalization"
+require "ffi-icu/normalizer"
 require "ffi-icu/break_iterator"
 require "ffi-icu/number_formatting"
 require "ffi-icu/time_formatting"
-

data/lib/ffi-icu/lib.rb

@@ -339,6 +339,13 @@ module ICU
 
     attach_function :unorm_normalize, "unorm_normalize#{suffix}", [:pointer, :int32_t, :normalization_mode, :int32_t, :pointer, :int32_t, :pointer], :int32_t
 
+    # http://icu-project.org/apiref/icu4c/unorm2_8h.html
+
+    enum :normalization2_mode, [ :compose, :decompose, :fcd, :compose_contiguous ]
+    attach_function :unorm2_getInstance, "unorm2_getInstance#{suffix}", [:pointer, :pointer, :normalization2_mode, :pointer], :pointer
+    attach_function :unorm2_normalize, "unorm2_normalize#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t, :pointer], :int32_t
+    attach_function :unorm2_isNormalized, "unorm2_isNormalized#{suffix}", [:pointer, :pointer, :int32_t, :pointer], :bool
+
     #
     # Text Boundary Analysis
     #
@@ -392,10 +399,10 @@ module ICU
       :ignore
     ]
     enum :number_format_attribute, [
-       :parse_int_only, :grouping_used, :decimal_always_show, :max_integer_digits, 
-       :min_integer_digits, :integer_digits, :max_fraction_digits, :min_fraction_digits, 
-       :fraction_digits, :multiplier, :grouping_size, :rounding_mode, 
-       :rounding_increment, :format_width, :padding_position, :secondary_grouping_size, 
+       :parse_int_only, :grouping_used, :decimal_always_show, :max_integer_digits,
+       :min_integer_digits, :integer_digits, :max_fraction_digits, :min_fraction_digits,
+       :fraction_digits, :multiplier, :grouping_size, :rounding_mode,
+       :rounding_increment, :format_width, :padding_position, :secondary_grouping_size,
        :significant_digits_used, :min_significant_digits, :max_significant_digits, :lenient_parse
     ]
     attach_function :unum_open, "unum_open#{suffix}", [:number_format_style, :pointer, :int32_t, :string, :pointer, :pointer ], :pointer

data/lib/ffi-icu/normalizer.rb

@@ -0,0 +1,47 @@
+module ICU
+  class Normalizer
+    # support for newer ICU normalization API
+
+    def initialize(package_name = nil, name = 'nfc', mode = :decompose)
+      Lib.check_error do |error|
+        @instance = Lib.unorm2_getInstance(package_name, name, mode, error)
+      end
+    end
+
+    def normalize(input)
+      input_length  = input.jlength
+      in_ptr        = UCharPointer.from_string(input)
+      needed_length = capacity = 0
+      out_ptr       = UCharPointer.new(needed_length)
+
+      retried = false
+      begin
+        Lib.check_error do |error|
+          needed_length = Lib.unorm2_normalize(@instance, in_ptr, input_length, out_ptr, capacity, error)
+        end
+      rescue BufferOverflowError
+        raise BufferOverflowError, "needed: #{needed_length}" if retried
+
+        capacity = needed_length
+        out_ptr = out_ptr.resized_to needed_length
+
+        retried = true
+        retry
+      end
+
+      out_ptr.string
+    end
+
+    def is_normailzed?(input)
+      input_length  = input.jlength
+      in_ptr        = UCharPointer.from_string(input)
+
+      Lib.check_error do |error|
+        result = Lib.unorm2_isNormalized(@instance, in_ptr, input_length, error)
+      end
+
+      result
+    end
+
+  end # Normalizer
+end # ICU

data/lib/ffi-icu/version.rb

@@ -1,3 +1,3 @@
 module ICU
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

data/spec/normalizer_spec.rb

@@ -0,0 +1,59 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+module ICU
+  describe Normalizer do
+    describe 'NFD: nfc decompose' do
+      let(:normalizer) { ICU::Normalizer.new(nil, 'nfc', :decompose) }
+
+      it "should normalize a string" do
+        normalizer.normalize("Å").unpack("U*").should == [65, 778]
+        normalizer.normalize("ô").unpack("U*").should == [111, 770]
+        normalizer.normalize("a").unpack("U*").should == [97]
+        normalizer.normalize("中文").unpack("U*").should == [20013, 25991]
+        normalizer.normalize("Äffin").unpack("U*").should == [65, 776, 102, 102, 105, 110]
+        normalizer.normalize("Äffin").unpack("U*").should == [65, 776, 64259, 110]
+        normalizer.normalize("Henry IV").unpack("U*").should == [72, 101, 110, 114, 121, 32, 73, 86]
+        normalizer.normalize("Henry Ⅳ").unpack("U*").should == [72, 101, 110, 114, 121, 32, 8547]
+      end
+    end
+
+    describe 'NFC: nfc compose' do
+      let(:normalizer) { ICU::Normalizer.new(nil, 'nfc', :compose) }
+
+      it "should normalize a string" do
+        normalizer.normalize("Å").unpack("U*").should == [197]
+        normalizer.normalize("ô").unpack("U*").should == [244]
+        normalizer.normalize("a").unpack("U*").should == [97]
+        normalizer.normalize("中文").unpack("U*").should == [20013, 25991]
+        normalizer.normalize("Äffin").unpack("U*").should == [196, 102, 102, 105, 110]
+        normalizer.normalize("Äffin").unpack("U*").should == [196, 64259, 110]
+        normalizer.normalize("Henry IV").unpack("U*").should == [72, 101, 110, 114, 121, 32, 73, 86]
+        normalizer.normalize("Henry Ⅳ").unpack("U*").should == [72, 101, 110, 114, 121, 32, 8547]
+      end
+    end
+
+    describe 'NFKD: nfkc decompose' do
+      let(:normalizer) { ICU::Normalizer.new(nil, 'nfkc', :decompose) }
+
+      it "should normalize a string" do
+        normalizer.normalize("Äffin").unpack("U*").should == [65, 776, 102, 102, 105, 110]
+        normalizer.normalize("Äffin").unpack("U*").should == [65, 776, 102, 102, 105, 110]
+        normalizer.normalize("Henry IV").unpack("U*").should == [72, 101, 110, 114, 121, 32, 73, 86]
+        normalizer.normalize("Henry Ⅳ").unpack("U*").should == [72, 101, 110, 114, 121, 32, 73, 86]
+      end
+    end
+
+    describe 'NFKC: nfkc compose' do
+      let(:normalizer) { ICU::Normalizer.new(nil, 'nfkc', :compose) }
+
+      it "should normalize a string" do
+        normalizer.normalize("Äffin").unpack("U*").should == [196, 102, 102, 105, 110]
+        normalizer.normalize("Äffin").unpack("U*").should == [196, 102, 102, 105, 110]
+        normalizer.normalize("Henry IV").unpack("U*").should == [72, 101, 110, 114, 121, 32, 73, 86]
+        normalizer.normalize("Henry Ⅳ").unpack("U*").should == [72, 101, 110, 114, 121, 32, 73, 86]
+      end
+    end
+  end # Normalizer
+end # ICU

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffi-icu
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Jari Bakken
@@ -86,6 +86,7 @@ files:
 - lib/ffi-icu/lib/util.rb
 - lib/ffi-icu/locale.rb
 - lib/ffi-icu/normalization.rb
+- lib/ffi-icu/normalizer.rb
 - lib/ffi-icu/number_formatting.rb
 - lib/ffi-icu/time_formatting.rb
 - lib/ffi-icu/transliteration.rb
@@ -98,6 +99,7 @@ files:
 - spec/lib_spec.rb
 - spec/locale_spec.rb
 - spec/normalization_spec.rb
+- spec/normalizer_spec.rb
 - spec/number_formatting_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
@@ -137,6 +139,7 @@ test_files:
 - spec/lib_spec.rb
 - spec/locale_spec.rb
 - spec/normalization_spec.rb
+- spec/normalizer_spec.rb
 - spec/number_formatting_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb