ffi-icu 0.6.1 → 0.8.0

data/lib/ffi-icu/normalization.rb

@@ -3,28 +3,28 @@
 module ICU
   module Normalization
     def self.normalize(input, mode = :default)
-      input_length  = input.size
       needed_length = out_length = options = 0
-      in_ptr        = UCharPointer.from_string(input)
+      input_pointer = UCharPointer.from_utf8(input)
+      input_length  = input_pointer.length_in_uchars
       out_ptr       = UCharPointer.new(out_length)
 
       retried = false
 
       begin
         Lib.check_error do |error|
-          needed_length = Lib.unorm_normalize(in_ptr, input_length, mode, options, out_ptr, out_length, error)
+          needed_length = Lib.unorm_normalize(input_pointer, input_length, mode, options, out_ptr, out_length, error)
         end
       rescue BufferOverflowError
         raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
-        out_ptr       = out_ptr.resized_to(needed_length)
         out_length    = needed_length + 1
+        out_ptr       = out_ptr.resized_to(out_length)
 
         retried = true
         retry
       end
 
-      out_ptr.string
+      out_ptr.utf8_string(needed_length)
     end
   end
 end

data/lib/ffi-icu/normalizer.rb

@@ -11,39 +11,38 @@ module ICU
     end
 
     def normalize(input)
-      input_length  = input.size
-      in_ptr        = UCharPointer.from_string(input)
+      input_pointer = UCharPointer.from_utf8(input)
+      input_length  = input_pointer.length_in_uchars
       needed_length = capacity = 0
       out_ptr       = UCharPointer.new(needed_length)
 
       retried = false
       begin
         Lib.check_error do |error|
-          needed_length = Lib.unorm2_normalize(@instance, in_ptr, input_length, out_ptr, capacity, error)
+          needed_length = Lib.unorm2_normalize(@instance, input_pointer, input_length, out_ptr, capacity, error)
         end
       rescue BufferOverflowError
         raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
-        capacity = needed_length
-        out_ptr = out_ptr.resized_to(needed_length)
+        capacity = needed_length + 1
+        out_ptr = out_ptr.resized_to(capacity)
 
         retried = true
         retry
       end
 
-      out_ptr.string
+      out_ptr.utf8_string(needed_length)
     end
 
-    def normailzed?(input)
-      input_length  = input.size
-      in_ptr        = UCharPointer.from_string(input)
+    def normalized?(input)
+      input_pointer = UCharPointer.from_utf8(input)
+      input_length  = input_pointer.length_in_uchars
 
       Lib.check_error do |error|
-        Lib.unorm2_isNormalized(@instance, in_ptr, input_length, error)
+        Lib.unorm2_isNormalized(@instance, input_pointer, input_length, error)
       end
-
-      result
     end
+    alias normailzed? normalized?
 
     def is_normalized?(input) # rubocop:disable Naming/PredicatePrefix
       Warning.warn('is_normalized? is deprecated and will be removed after v0.7. Please use normalized? instead.')

data/lib/ffi-icu/number_formatting.rb

@@ -15,6 +15,10 @@ module ICU
       end
     end
 
+    def self.create_with_skeleton(locale, skeleton)
+      SkeletonFormatter.new(locale, skeleton)
+    end
+
     def self.clear_default_options
       @default_options.clear
     end
@@ -35,6 +39,10 @@ module ICU
       create(locale, :currency, options).format(number, currency)
     end
 
+    def self.format_with_skeleton(locale, number, skeleton)
+      create_with_skeleton(locale, skeleton).format(number)
+    end
+
     def self.spell(locale, number, options = {})
       create(locale, :spellout, options).format(number)
     end
@@ -48,6 +56,11 @@ module ICU
       private
 
       def make_formatter(type, locale)
+        if type == :pattern_rule_based
+          raise(ArgumentError, 'The pattern_rule_based style needs custom rules, which ffi-icu does not support; ' \
+                               'ICU::NumberFormatting.create_with_skeleton covers most other formats')
+        end
+
         ptr = Lib.check_error do |error|
           Lib.unum_open(type, FFI::MemoryPointer.new(4), 0, locale, FFI::MemoryPointer.new(4), error)
         end
@@ -97,7 +110,7 @@ module ICU
                               end
             end
           end
-          out_ptr.string(needed_length)
+          out_ptr.utf8_string(needed_length)
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -136,10 +149,10 @@ module ICU
 
         begin
           Lib.check_error do |error|
-            needed_length = Lib.unum_format_currency(@f, number, UCharPointer.from_string(currency, 4), out_ptr,
+            needed_length = Lib.unum_format_currency(@f, number, UCharPointer.from_utf8(currency, 4), out_ptr,
                                                      needed_length, nil, error)
           end
-          out_ptr.string
+          out_ptr.utf8_string
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -149,5 +162,33 @@ module ICU
         end
       end
     end
+
+    class SkeletonFormatter
+      def initialize(locale, skeleton)
+        unless Lib.respond_to?(:unumf_openForSkeletonAndLocale)
+          raise('ICU::NumberFormatting::SkeletonFormatter requires ICU >= 62')
+        end
+
+        skeleton_uchar = UCharPointer.from_utf8(skeleton)
+        @f = FFI::AutoPointer.new(
+          Lib.check_error do |error|
+            Lib.unumf_openForSkeletonAndLocale(skeleton_uchar, skeleton_uchar.length_in_uchars, locale, error)
+          end,
+          Lib.method(:unumf_close)
+        )
+      end
+
+      def format(number)
+        decimal = number.to_s
+        result = FFI::AutoPointer.new(
+          Lib.check_error { |error| Lib.unumf_openResult(error) },
+          Lib.method(:unumf_closeResult)
+        )
+        Lib.check_error { |error| Lib.unumf_formatDecimal(@f, decimal, decimal.bytesize, result, error) }
+        Lib::Util.read_uchar_buffer(0) do |buffer, error|
+          Lib.unumf_resultToString(result, buffer, buffer.length_in_uchars, error)
+        end
+      end
+    end
   end
 end

data/lib/ffi-icu/time_formatting.rb

@@ -91,8 +91,8 @@ module ICU
         pattern_ptr = FFI::MemoryPointer.new(4)
 
         if time_zone_str
-          time_zone = UCharPointer.from_string(time_zone_str)
-          tz_len = time_zone_str.size
+          time_zone = UCharPointer.from_utf8(time_zone_str)
+          tz_len = time_zone.length_in_uchars
         else
           Lib.check_error do |error|
             i_len = 150
@@ -142,8 +142,8 @@ module ICU
       end
 
       def parse(str)
-        str_u = UCharPointer.from_string(str)
-        str_l = str.size
+        str_u = UCharPointer.from_utf8(str)
+        str_l = str_u.length_in_uchars
         Lib.check_error do |error|
           ret = Lib.udat_parse(@f, str_u, str_l, nil, error)
           Time.at(ret / 1000.0)
@@ -170,7 +170,7 @@ module ICU
             end
           end
 
-          out_ptr.string
+          out_ptr.utf8_string
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -209,7 +209,7 @@ module ICU
             needed_length = Lib.udat_toPattern(@f, localized, out_ptr, needed_length, error)
           end
 
-          out_ptr.string
+          out_ptr.utf8_string
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -228,8 +228,8 @@ module ICU
       end
 
       def skeleton_format(skeleton_pattern_str, locale)
-        skeleton_pattern_ptr = UCharPointer.from_string(skeleton_pattern_str)
-        skeleton_pattern_len = skeleton_pattern_str.size
+        skeleton_pattern_ptr = UCharPointer.from_utf8(skeleton_pattern_str)
+        skeleton_pattern_len = skeleton_pattern_ptr.length_in_uchars
 
         needed_length = 0
         pattern_ptr = UCharPointer.new(needed_length)
@@ -272,7 +272,7 @@ module ICU
         return unless @hour_cycle
 
         # Get the current pattern and convert to a skeleton
-        skeleton_str = pattern_to_skeleton_uchar(current_pattern_as_uchar).string
+        skeleton_str = pattern_to_skeleton_uchar(current_pattern_as_uchar).utf8_string
 
         # Manipulate the skeleton to make it work with the correct hour cycle.
         skeleton_str.gsub!(/[hHkKjJ]/, HOUR_CYCLE_SYMS[@hour_cycle])
@@ -286,7 +286,7 @@ module ICU
         end
 
         # Convert the skeleton back to a pattern
-        new_pattern_str = skeleton_to_pattern_uchar(UCharPointer.from_string(skeleton_str)).string
+        new_pattern_str = skeleton_to_pattern_uchar(UCharPointer.from_utf8(skeleton_str)).utf8_string
 
         # We also need to manipulate the _pattern_, a little bit, because (according to Firefox source):
         #
@@ -345,8 +345,8 @@ module ICU
       end
 
       def set_date_format_impl(localized, pattern_str)
-        pattern     = UCharPointer.from_string(pattern_str)
-        pattern_len = pattern_str.size
+        pattern     = UCharPointer.from_utf8(pattern_str)
+        pattern_len = pattern.length_in_uchars
 
         Lib.check_error do |_error|
           Lib.udat_applyPattern(@f, localized, pattern, pattern_len)

data/lib/ffi-icu/transliteration.rb

@@ -26,15 +26,17 @@ module ICU
         rules_length = 0
 
         if rules
-          rules_length = rules.size + 1
-          rules = UCharPointer.from_string(rules)
+          rules = UCharPointer.from_utf8(rules)
+          rules_length = rules.length_in_uchars
         end
 
+        id = UCharPointer.from_utf8(id)
+
         parse_error = Lib::UParseError.new
         begin
           Lib.check_error do |status|
-            ptr = Lib.utrans_openU(UCharPointer.from_string(id), id.size, direction, rules, rules_length,
-                                   @parse_error, status)
+            ptr = Lib.utrans_openU(id, id.length_in_uchars, direction, rules, rules_length,
+                                   parse_error, status)
             @tr = FFI::AutoPointer.new(ptr, Lib.method(:utrans_close))
           end
         rescue ICU::Error => e
@@ -45,9 +47,11 @@ module ICU
       def transliterate(from)
         # this is a bit unpleasant
 
-        unicode_size = from.unpack('U*').size
-        capacity     = unicode_size + 1
-        buf          = UCharPointer.from_string(from, capacity)
+        input = UCharPointer.from_utf8(from)
+        input_length_in_uchars = input.length_in_uchars
+        capacity               = input_length_in_uchars + 1
+        buf = UCharPointer.new(capacity)
+        buf.put_bytes(0, input.get_bytes(0, input.size))
         limit        = FFI::MemoryPointer.new(:int32)
         text_length  = FFI::MemoryPointer.new(:int32)
 
@@ -56,7 +60,7 @@ module ICU
         begin
           # resets to original size on retry
           [limit, text_length].each do |ptr|
-            ptr.put_int32(0, unicode_size)
+            ptr.put_int32(0, input_length_in_uchars)
           end
 
           Lib.check_error do |error|
@@ -73,13 +77,13 @@ module ICU
           # create a new buffer with more capacity instead of resizing,
           # since the old buffer now has result data
           buf.free
-          buf = UCharPointer.from_string(from, capacity)
+          buf = UCharPointer.from_utf8(from, capacity)
 
           retried = true
           retry
         end
 
-        buf.string(text_length.get_int32(0))
+        buf.utf8_string(text_length.get_int32(0))
       end
     end
   end

data/lib/ffi-icu/uchar.rb

@@ -22,6 +22,22 @@ module ICU
       ptr
     end
 
+    def self.from_utf8(str, capacity = nil)
+      chars = str.encode(Encoding::UTF_16LE).unpack('v*')
+
+      if capacity
+        raise(ArgumentError, "capacity is too small for string of #{chars.size} UChars") if capacity < chars.size
+
+        ptr = new(capacity)
+      else
+        ptr = new(chars.size)
+      end
+
+      ptr.write_array_of_uint16(chars)
+
+      ptr
+    end
+
     def initialize(size)
       super(UCHAR_TYPE, size)
     end
@@ -42,6 +58,13 @@ module ICU
       wstring.pack('U*')
     end
 
+    def utf8_string(length = nil)
+      length ||= size / TYPE_SIZE
+
+      wstring = read_array_of_uint16(length).pack('v*')
+      wstring.force_encoding(Encoding::UTF_16LE).encode(Encoding::UTF_8)
+    end
+
     def length_in_uchars
       size / type_size
     end

data/lib/ffi-icu/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ICU
-  VERSION = '0.6.1'
+  VERSION = '0.8.0'
 end

data/lib/ffi-icu.rb

@@ -25,11 +25,14 @@ require 'ffi-icu/lib/util'
 require 'ffi-icu/uchar'
 require 'ffi-icu/chardet'
 require 'ffi-icu/collation'
+require 'ffi-icu/currency'
 require 'ffi-icu/locale'
 require 'ffi-icu/transliteration'
 require 'ffi-icu/normalization'
 require 'ffi-icu/normalizer'
 require 'ffi-icu/break_iterator'
 require 'ffi-icu/number_formatting'
+require 'ffi-icu/message_formatting'
 require 'ffi-icu/time_formatting'
 require 'ffi-icu/duration_formatting'
+require 'ffi-icu/list_formatting'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffi-icu
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Erick Guan
@@ -76,10 +76,13 @@ files:
 - lib/ffi-icu/break_iterator.rb
 - lib/ffi-icu/chardet.rb
 - lib/ffi-icu/collation.rb
+- lib/ffi-icu/currency.rb
 - lib/ffi-icu/duration_formatting.rb
 - lib/ffi-icu/lib.rb
 - lib/ffi-icu/lib/util.rb
+- lib/ffi-icu/list_formatting.rb
 - lib/ffi-icu/locale.rb
+- lib/ffi-icu/message_formatting.rb
 - lib/ffi-icu/normalization.rb
 - lib/ffi-icu/normalizer.rb
 - lib/ffi-icu/number_formatting.rb