ffi-icu 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f219be994f4a69799cf8b848c0909786476face806e223c89f8f4b455ef9b85
-  data.tar.gz: 562ceb9c82764d4251566107cadaabe7d0f63c69f012a0771c6ebf2d75677252
+  metadata.gz: 6ffb113b9b98bd46355e3d6750fc1026d7732d06a8575bf2022aa073e72c532e
+  data.tar.gz: 43b5a202de5a08790c2a257a8a05f9867df2a5b42624cab157dbf20eaa85eb37
 SHA512:
-  metadata.gz: ca6afe1397a8879f89303ed2b9866caaa906d0b4271b4fb75f8eb8a51e8e9a482f8b398072f13a85e5d790e9b261a7ab3178dda2c2a35a5dceb575fd798b5679
-  data.tar.gz: 9f722bd3b1248aeac053d74230ce9c98a9cde2d1e925a86284179880d969ec7d1fe59dbabf579306e9f63101bb013198ba6c637e4aa7e561a48b7995451e4411
+  metadata.gz: 63fefe8619f8bfbad7ead9ac1ada9471074246b1a2d7c4af6cdb899250f464a07c8596bb55e836f44a128d1ce077b939404b2ad85c612298cc65a66bc1fe92f8
+  data.tar.gz: 06e98c3af40a060bbbd2a19a9e5b95fcea49d1d75316a1f2ea639d91d88721886fe079ad3e4952ac8750cdfbf62e5dc4550b0893c3583c1896764c46d1c3fd46

data/README.md

@@ -93,6 +93,28 @@ curf = ICU::NumberFormatting.create('en-US', :currency)
 curf.format(1234.56, 'USD') #=> "$1,234.56"
 ```
 
+## Message Formatting
+
+Message formatting supports ICU MessageFormat patterns using numbered arguments:
+
+```ruby
+formatter = ICU::MessageFormatting.create(
+  '{0, plural, one {# item} other {# items}}',
+  locale: 'en-US'
+)
+
+formatter.format([2]) #=> "2 items"
+```
+
+Patterns can also be validated without formatting a message:
+
+```ruby
+ICU::MessageFormatting.validate('{0, select, male {He} female {She} other {They}}', locale: 'en-US')
+#=> true
+```
+
+The current API uses ICU's C MessageFormat API and therefore resolves numbered arguments only. Named arguments are not resolved by this API, and rich-text tags require a separate compatibility layer. Supported Ruby argument types are `String`, `Integer`, `Float`, and `BigDecimal`.
+
 ## Time Formatting/Parsing
 
 Examples:
@@ -187,6 +209,26 @@ Example:
 ICU::Transliteration.transliterate('Traditional-Simplified', '沈從文') # => "沈从文"
 ```
 
+## List Formatting
+
+Join items with locale-aware separators and conjunctions:
+
+```ruby
+ICU::ListFormatting.format(['Alice', 'Bob', 'Jane'], locale: 'en', style: :and)
+# => "Alice, Bob, and Jane"
+
+ICU::ListFormatting.format(['Alice', 'Bob', 'Jane'], locale: 'fr', style: :and)
+# => "Alice, Bob et Jane"
+```
+
+Supported styles are:
+
+- `:and`: the default
+- `:or`
+- `:unit`: ICU's units list type
+
+Only ICU's wide width is exposed. Styles must be symbols, and items must be strings. The Ruby/FFI layer converts Ruby strings to ICU's UTF-16 `UChar` values before calling ICU. List formatting requires ICU 67 or newer.
+
 ## Locale
 
 Examples:
@@ -197,7 +239,10 @@ locale.display_country('en-US') #=> "United States"
 locale.display_language('es') #=> "inglés"
 locale.display_name('es') #=> "inglés (Estados Unidos)"
 locale.display_name_with_context('en-US', [:length_short]) #=> "English (US)"
-locale.display_name_with_context('en-US', [:length_long])  #=> "English (United States)"
+locale.display_name_with_context('en-US', [:length_full])  #=> "English (United States)"
+
+dari = ICU::Locale.new('fa-AF')
+dari.display_name_with_context('en', [:dialect_names]) #=> "Dari"
 ```
 
 [icu]: https://github.com/unicode-org/icu

data/lib/ffi-icu/break_iterator.rb

@@ -20,16 +20,20 @@ module ICU
     end
 
     def text=(str)
-      @text = str
+      text_pointer = UCharPointer.from_utf8(str)
 
       Lib.check_error do |err|
-        Lib.ubrk_setText(@iterator, UCharPointer.from_string(str), str.size, err)
+        Lib.ubrk_setText(@iterator, text_pointer, text_pointer.length_in_uchars, err)
       end
+
+      @text = str
+      @text_pointer = text_pointer
     end
 
     def each
       return to_enum(:each) unless block_given?
 
+      # Positional methods expose ICU UTF-16 code-unit offsets.
       int = first
 
       while int != DONE
@@ -43,11 +47,13 @@ module ICU
     def each_substring
       return to_enum(:each_substring) unless block_given?
 
-      # each_char needed for 1.8, where String#[] works on bytes, not characters
       chars = text.each_char.to_a
-      low   = first
+      utf16_to_ruby = ruby_offsets_for(chars)
+
+      low = utf16_to_ruby[first]
 
       while (high = self.next) != DONE
+        high = utf16_to_ruby[high]
         yield(chars[low...high].join)
         low = high
       end
@@ -55,6 +61,19 @@ module ICU
       self
     end
 
+    def ruby_offsets_for(chars)
+      offsets = [0]
+      utf16_offset = 0
+
+      chars.each_with_index do |char, ruby_index|
+        utf16_offset += char.encode(Encoding::UTF_16LE).bytesize / UCharPointer::TYPE_SIZE
+        offsets[utf16_offset] = ruby_index + 1
+      end
+
+      offsets
+    end
+    private :ruby_offsets_for
+
     def substrings
       each_substring.to_a
     end
@@ -64,7 +83,7 @@ module ICU
     end
 
     def previous
-      Lib.ubrk_next(@iterator)
+      Lib.ubrk_previous(@iterator)
     end
 
     def first

data/lib/ffi-icu/collation.rb

@@ -72,21 +72,15 @@ module ICU
       end
 
       def compare(a, b)
-        Lib.ucol_strcoll(
-          @c,
-          UCharPointer.from_string(a), a.size,
-          UCharPointer.from_string(b), b.size
-        )
+        Lib.ucol_strcoll(@c, *collation_arguments(a, b))
       end
 
       def greater?(a, b)
-        Lib.ucol_greater(@c, UCharPointer.from_string(a), a.size,
-                         UCharPointer.from_string(b), b.size)
+        Lib.ucol_greater(@c, *collation_arguments(a, b))
       end
 
       def greater_or_equal?(a, b)
-        Lib.ucol_greaterOrEqual(@c, UCharPointer.from_string(a), a.size,
-                                UCharPointer.from_string(b), b.size)
+        Lib.ucol_greaterOrEqual(@c, *collation_arguments(a, b))
       end
 
       def equal?(*args)
@@ -96,8 +90,7 @@ module ICU
 
         a, b = args
 
-        Lib.ucol_equal(@c, UCharPointer.from_string(a), a.size,
-                       UCharPointer.from_string(b), b.size)
+        Lib.ucol_equal(@c, *collation_arguments(a, b))
       end
 
       def collate(sortable)
@@ -110,15 +103,16 @@ module ICU
         @rules ||= begin
           length = FFI::MemoryPointer.new(:int)
           ptr = Lib.ucol_getRules(@c, length)
-          ptr.read_array_of_uint16(length.read_int).pack('U*')
+          ptr.read_array_of_uint16(length.read_int).
+            pack('v*').force_encoding('UTF-16LE').encode('UTF-8')
         end
       end
 
       def collation_key(string)
-        ptr = UCharPointer.from_string(string)
-        size = Lib.ucol_getSortKey(@c, ptr, string.size, nil, 0)
+        ptr = UCharPointer.from_utf8(string)
+        size = Lib.ucol_getSortKey(@c, ptr, ptr.length_in_uchars, nil, 0)
         buffer = FFI::MemoryPointer.new(:char, size)
-        Lib.ucol_getSortKey(@c, ptr, string.size, buffer, size)
+        Lib.ucol_getSortKey(@c, ptr, ptr.length_in_uchars, buffer, size)
         buffer.read_bytes(size - 1)
       end
 
@@ -134,6 +128,17 @@ module ICU
         end
       end
 
+      def collation_arguments(left, right)
+        left_text = UCharPointer.from_utf8(left)
+        right_text = UCharPointer.from_utf8(right)
+
+        [
+          left_text, left_text.length_in_uchars,
+          right_text, right_text.length_in_uchars
+        ]
+      end
+      private :collation_arguments
+
       # create friendly named methods for setting attributes
       ATTRIBUTES.each_key do |attribute|
         class_eval <<-CODE, __FILE__, __LINE__ + 1

data/lib/ffi-icu/currency.rb

@@ -0,0 +1,123 @@
+module ICU
+  class Currency
+    CURRENCY_TYPE_COMMON         = 1
+    CURRENCY_TYPE_UNCOMMON       = 2
+    CURRENCY_TYPE_DEPRECATED     = 4
+    CURRENCY_TYPE_NON_DEPRECATED = 8
+    CURRENCY_TYPE_ALL = (2**31) - 1
+
+    class << self
+      # Returns the default currency code for a locale, e.g. "USD" for "en_US"
+      def for_locale(locale)
+        ptr = UCharPointer.new(4)
+        length = Lib.check_error do |status|
+          Lib.ucurr_forLocale(locale.to_s, ptr, 4, status)
+        end
+        ptr.utf8_string(length)
+      end
+
+      # Returns an array of ISO 4217 currency code strings.
+      # currency_type: :all (default), :common, :uncommon, :deprecated, :non_deprecated
+      def available(currency_type = :all)
+        flag = case currency_type
+               when :common         then CURRENCY_TYPE_COMMON
+               when :uncommon       then CURRENCY_TYPE_UNCOMMON
+               when :deprecated     then CURRENCY_TYPE_DEPRECATED
+               when :non_deprecated then CURRENCY_TYPE_NON_DEPRECATED
+               else CURRENCY_TYPE_ALL
+               end
+
+        enum_ptr = Lib.check_error { |status| Lib.ucurr_openISOCurrencies(flag, status) }
+        begin
+          Lib.enum_ptr_to_array(enum_ptr)
+        ensure
+          Lib.uenum_close(enum_ptr)
+        end
+      end
+
+      # Convenience: ICU::Currency.symbol('USD', 'en_US') => "$"
+      def symbol(code, locale)
+        new(code).symbol(locale)
+      end
+
+      # Convenience: ICU::Currency.name('USD', 'en') => "US Dollar"
+      def name(code, locale)
+        new(code).name(locale)
+      end
+    end
+
+    attr_reader :code
+
+    def initialize(code)
+      @code = code.to_s.upcase
+    end
+
+    # Currency symbol for the given locale, e.g. "$", "€"
+    def symbol(locale)
+      get_name(locale, :symbol_name)
+    end
+
+    # Narrow (shortest unambiguous) symbol. Requires ICU >= 61.
+    def narrow_symbol(locale)
+      raise('narrow_symbol requires ICU >= 61') if Lib.version.to_a.first < 61
+
+      get_name(locale, :narrow_symbol_name)
+    end
+
+    # Long display name for the given locale, e.g. "US Dollar"
+    def name(locale)
+      get_name(locale, :long_name)
+    end
+
+    # Plural form of the currency name.
+    # plural_count: "zero", "one", "two", "few", "many", "other"
+    def plural_name(locale, plural_count = 'other')
+      raise('plural_name requires ICU >= 4.2') unless Lib.respond_to?(:ucurr_getPluralName)
+
+      is_choice = FFI::MemoryPointer.new(:uint8)
+      len       = FFI::MemoryPointer.new(:int32_t)
+
+      ptr = Lib.check_error do |status|
+        Lib.ucurr_getPluralName(currency_uchar, locale.to_s, is_choice, plural_count.to_s, len, status)
+      end
+      ptr.read_array_of_uint16(len.read_int32).pack('U*')
+    end
+
+    # Number of decimal digits used when displaying this currency.
+    # usage: :standard (default) or :cash
+    def fraction_digits(usage = :standard)
+      if usage == :cash && Lib.respond_to?(:ucurr_getDefaultFractionDigitsForUsage)
+        Lib.check_error { |status| Lib.ucurr_getDefaultFractionDigitsForUsage(currency_uchar, :cash, status) }
+      else
+        Lib.check_error { |status| Lib.ucurr_getDefaultFractionDigits(currency_uchar, status) }
+      end
+    end
+
+    # ISO 4217 numeric code, e.g. 840 for USD. Requires ICU >= 49.
+    def numeric_code
+      raise('numeric_code requires ICU >= 49') unless Lib.respond_to?(:ucurr_getNumericCode)
+
+      Lib.ucurr_getNumericCode(currency_uchar)
+    end
+
+    def to_s
+      @code
+    end
+
+    private
+
+    def currency_uchar
+      UCharPointer.from_utf8(@code, 4)
+    end
+
+    def get_name(locale, style)
+      is_choice = FFI::MemoryPointer.new(:uint8)
+      len       = FFI::MemoryPointer.new(:int32_t)
+
+      ptr = Lib.check_error do |status|
+        Lib.ucurr_getName(currency_uchar, locale.to_s, style, is_choice, len, status)
+      end
+      ptr.read_array_of_uint16(len.read_int32).pack('U*')
+    end
+  end
+end

data/lib/ffi-icu/duration_formatting.rb

@@ -264,7 +264,7 @@ module ICU
 
       def number_formatter(skeleton)
         @number_formatters[skeleton] ||= begin
-          skeleton_uchar = UCharPointer.from_string(skeleton)
+          skeleton_uchar = UCharPointer.from_utf8(skeleton)
           FFI::AutoPointer.new(
             Lib.check_error do |error|
               Lib.unumf_openForSkeletonAndLocale(skeleton_uchar, skeleton_uchar.length_in_uchars,
@@ -291,7 +291,7 @@ module ICU
       end
 
       def format_list(values)
-        value_uchars = values.map(&UCharPointer.method(:from_string))
+        value_uchars = values.map(&UCharPointer.method(:from_utf8))
         value_uchars_array = FFI::MemoryPointer.new(:pointer, value_uchars.size)
         value_uchars_array.put_array_of_pointer(0, value_uchars)
         value_lengths_array = FFI::MemoryPointer.new(:int32_t, value_uchars.size)

data/lib/ffi-icu/lib/util.rb

@@ -32,7 +32,7 @@ module ICU
 
       def self.read_uchar_buffer(length, &)
         buf, len = read_uchar_buffer_as_ptr_impl(length, &)
-        buf.string(len)
+        buf.utf8_string(len)
       end
 
       def self.read_uchar_buffer_as_ptr(length, &)

data/lib/ffi-icu/lib.rb

@@ -19,18 +19,23 @@ module ICU
                         elsif FFI::Platform::IS_WINDOWS
                           ENV['PATH'].split(File::PATH_SEPARATOR)
                         else
+                          multiarch_paths = Dir['/usr/lib/*-linux-gnu']
+                          multiarch_paths.sort_by! do |path|
+                            File.basename(path).start_with?("#{FFI::Platform::ARCH}-") ? 0 : 1
+                          end
+
                           [
                             '/usr/local/{lib64,lib}',
                             '/opt/local/{lib64,lib}',
                             '/usr/{lib64,lib}'
-                          ] + Dir['/usr/lib/*-linux-gnu'] # for Debian Multiarch http://wiki.debian.org/Multiarch
+                          ] + multiarch_paths # for Debian Multiarch http://wiki.debian.org/Multiarch
                         end
     end
 
     def self.find_lib(lib)
-      Dir.glob(search_paths.map do |path|
-        File.expand_path(File.join(path, lib))
-      end).first
+      search_paths.lazy.filter_map do |path|
+        Dir.glob(File.expand_path(File.join(path, lib))).first
+      end.first
     end
 
     def self.load_icu
@@ -120,6 +125,7 @@ module ICU
 
     def self.check_error
       ptr = FFI::MemoryPointer.new(:int)
+      ptr.write_int(0)
       ret = yield(ptr)
       error_code = ptr.read_int
 
@@ -203,6 +209,10 @@ module ICU
 
     typedef VersionInfo, :version
 
+    attach_function :u_getVersion,      "u_getVersion#{suffix}",      [:version], :void
+    attach_function :u_versionToString, "u_versionToString#{suffix}", [:version, :pointer], :void
+    version ||= self.version.to_s
+
     attach_function :u_errorName,     "u_errorName#{suffix}",     [:int], :string
     attach_function :uenum_count,     "uenum_count#{suffix}",     [:pointer, :pointer], :int
     attach_function :uenum_close,     "uenum_close#{suffix}",     [:pointer], :void
@@ -210,9 +220,6 @@ module ICU
     attach_function :u_charsToUChars, "u_charsToUChars#{suffix}", [:string, :pointer, :int32_t], :void
     attach_function :u_UCharsToChars, "u_UCharsToChars#{suffix}", [:pointer, :string, :int32_t], :void
 
-    attach_function :u_getVersion,      "u_getVersion#{suffix}",      [:version], :void
-    attach_function :u_versionToString, "u_versionToString#{suffix}", [:version, :pointer], :void
-
     #
     # Locale
     #
@@ -345,16 +352,51 @@ module ICU
     attach_function :ucol_getAttribute,     "ucol_getAttribute#{suffix}",     [:pointer, :int, :pointer], :int
     attach_function :ucol_setAttribute,     "ucol_setAttribute#{suffix}",     [:pointer, :int, :int, :pointer], :void
 
+    # Currency
+    #
+    # https://unicode-org.github.io/icu-docs/apidoc/dev/icu4c/ucurr_8h.html
+    #
+
+    enum :ucurr_name_style, [
+      :symbol_name,         0,   # "$"
+      :long_name,           1,   # "US Dollar"
+      :narrow_symbol_name,  4,   # shortest unambiguous symbol (ICU >= 61)
+      :formal_symbol_name,  5,   # banking context (ICU >= 68)
+      :variant_symbol_name, 6    # regional variant (ICU >= 68)
+    ]
+
+    enum :ucurrency_usage, [
+      :standard, 0,  # UCURR_USAGE_STANDARD
+      :cash,     1   # UCURR_USAGE_CASH
+    ]
+
+    attach_function :ucurr_getName, "ucurr_getName#{suffix}",
+                    [:pointer, :string, :ucurr_name_style, :pointer, :pointer, :pointer], :pointer
+    attach_optional_function :ucurr_getPluralName, "ucurr_getPluralName#{suffix}",
+                             [:pointer, :string, :pointer, :string, :pointer, :pointer], :pointer
+    attach_function :ucurr_forLocale, "ucurr_forLocale#{suffix}",
+                    [:string, :pointer, :int32_t, :pointer], :int32_t
+    attach_function :ucurr_openISOCurrencies, "ucurr_openISOCurrencies#{suffix}",
+                    [:uint32, :pointer], :pointer
+    attach_function :ucurr_getDefaultFractionDigits, "ucurr_getDefaultFractionDigits#{suffix}",
+                    [:pointer, :pointer], :int32_t
+    attach_optional_function :ucurr_getDefaultFractionDigitsForUsage, "ucurr_getDefaultFractionDigitsForUsage#{suffix}",
+                             [:pointer, :ucurrency_usage, :pointer], :int32_t
+    attach_optional_function :ucurr_getNumericCode, "ucurr_getNumericCode#{suffix}",
+                             [:pointer], :int32_t
+
     # Transliteration
     #
     # http://icu-project.org/apiref/icu4c/utrans_8h.html
     #
 
     class UParseError < FFI::Struct
+      U_PARSE_CONTEXT_LEN = 16
+
       layout :line, :int32_t,
              :offset,       :int32_t,
-             :pre_context,  :pointer,
-             :post_context, :pointer
+             :pre_context,  [:uint16, U_PARSE_CONTEXT_LEN],
+             :post_context, [:uint16, U_PARSE_CONTEXT_LEN]
 
       def to_s
         format('#<%<class>s:%<hash>x line: %<line>d offset: %<offset>d',
@@ -362,6 +404,17 @@ module ICU
       end
     end
 
+    # MessageFormat
+    #
+    # https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/umsg_8h.html
+    #
+
+    attach_function :umsg_open, "umsg_open#{suffix}",
+                    [:pointer, :int32_t, :string, :pointer, :pointer], :pointer
+    attach_function :umsg_close, "umsg_close#{suffix}", [:pointer], :void
+    attach_function :umsg_format, "umsg_format#{suffix}",
+                    [:pointer, :pointer, :int32_t, :pointer, :varargs], :int32_t
+
     class UTransPosition < FFI::Struct
       layout :context_start, :int32_t,
              :context_limit, :int32_t,

data/lib/ffi-icu/list_formatting.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module ICU
+  module ListFormatting
+    VALID_STYLES = [:and, :or, :unit].freeze
+
+    STYLES_TO_TYPES = {
+      and: :and,
+      or: :or,
+      unit: :units
+    }.freeze
+
+    def self.format(items, locale:, style: :and)
+      raise(ArgumentError, "Unknown style #{style}") unless VALID_STYLES.include?(style)
+      raise(ArgumentError, 'items must be strings') unless items.all?(String)
+
+      return '' if items.empty?
+
+      unless Lib.respond_to?(:ulistfmt_openForType) &&
+             Lib.respond_to?(:ulistfmt_close) &&
+             Lib.respond_to?(:ulistfmt_format)
+        raise('ICU::ListFormatting requires ICU >= 67')
+      end
+
+      formatter = Lib.check_error do |error|
+        Lib.ulistfmt_openForType(locale, STYLES_TO_TYPES.fetch(style), :wide, error)
+      end
+
+      begin
+        value_uchars = items.map { |item| UCharPointer.from_utf8(item) }
+        value_uchars_array = FFI::MemoryPointer.new(:pointer, value_uchars.size)
+        value_uchars_array.put_array_of_pointer(0, value_uchars)
+        value_lengths_array = FFI::MemoryPointer.new(:int32_t, value_uchars.size)
+        value_lengths_array.put_array_of_int32(0, value_uchars.map(&:length_in_uchars))
+
+        Lib::Util.read_uchar_buffer(0) do |buffer, error|
+          Lib.ulistfmt_format(
+            formatter,
+            value_uchars_array,
+            value_lengths_array,
+            value_uchars.size,
+            buffer,
+            buffer.length_in_uchars,
+            error
+          )
+        end
+      ensure
+        Lib.ulistfmt_close(formatter)
+      end
+    end
+  end
+end

data/lib/ffi-icu/locale.rb

@@ -45,6 +45,7 @@ module ICU
     attr_reader :id
 
     DISPLAY_CONTEXT = {
+      dialect_names: 1, # UDISPCTX_DIALECT_NAMES = (UDISPCTX_TYPE_DIALECT_HANDLING<<8) + 1
       length_full: 512, # UDISPCTX_LENGTH_FULL  = (UDISPCTX_TYPE_DISPLAY_LENGTH<<8) + 0
       length_short: 513 # UDISPCTX_LENGTH_SHORT = (UDISPCTX_TYPE_DISPLAY_LENGTH<<8) + 1
     }.freeze

data/lib/ffi-icu/message_formatting.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'bigdecimal'
+
+module ICU
+  module MessageFormatting
+    class MessageFormatError < ICU::Error
+    end
+
+    class InvalidPatternError < MessageFormatError
+      attr_reader :locale, :pattern, :parse_error
+
+      def initialize(message, locale:, pattern:, parse_error:)
+        super(message)
+        @locale = locale
+        @pattern = pattern
+        @parse_error = parse_error
+      end
+
+      def offset
+        parse_error[:offset]
+      end
+
+      def line
+        parse_error[:line]
+      end
+    end
+
+    def self.create(pattern, locale:)
+      Formatter.new(pattern, locale:)
+    end
+
+    def self.format(pattern, locale:, arguments:)
+      create(pattern, locale:).format(arguments)
+    end
+
+    # rubocop:disable-next Naming/PredicateMethod
+    def self.validate(pattern, locale:)
+      create(pattern, locale:)
+      true
+    end
+
+    def self.valid?(pattern, locale:)
+      validate(pattern, locale:)
+    rescue InvalidPatternError
+      false
+    end
+
+    class Formatter
+      attr_reader :locale, :pattern
+
+      def initialize(pattern, locale:)
+        @pattern = String(pattern)
+        @locale = String(locale)
+        @pattern_pointer = UCharPointer.from_utf8(@pattern)
+        @formatter = open_formatter
+      end
+
+      def format(arguments = [])
+        validate_argument_count!(arguments)
+        typed_arguments = typed_arguments_for(arguments)
+        validate_argument_types!(arguments)
+
+        result = Lib::Util.read_uchar_buffer_as_ptr(0) do |result, status|
+          Lib.umsg_format(
+            @formatter,
+            result,
+            result.length_in_uchars,
+            status,
+            *typed_arguments
+          )
+        end
+        result.utf8_string
+      rescue ICU::Error => e
+        raise MessageFormatError, "could not format message: #{e.message}"
+      end
+
+      private
+
+      def open_formatter
+        parse_error = Lib::UParseError.new
+        status = FFI::MemoryPointer.new(:int32_t)
+        status.write_int32(0)
+        pointer = Lib.umsg_open(
+          @pattern_pointer,
+          @pattern_pointer.length_in_uchars,
+          @locale,
+          parse_error,
+          status
+        )
+
+        error_code = status.read_int32
+        return FFI::AutoPointer.new(pointer, Lib.method(:umsg_close)) unless error_code.positive?
+
+        raise InvalidPatternError.new(
+          "invalid message pattern: #{Lib.u_errorName(error_code)} " \
+          "at line #{parse_error[:line]}, offset #{parse_error[:offset]}",
+          locale: @locale,
+          pattern: @pattern,
+          parse_error: parse_error
+        )
+      end
+
+      def typed_arguments_for(arguments)
+        unless arguments.is_a?(Array)
+          raise ArgumentError, 'MessageFormat arguments must be provided as an Array of numbered values'
+        end
+
+        arguments.flat_map do |argument|
+          case argument
+          when String
+            [:pointer, null_terminated_uchar_pointer(argument)]
+          when Integer, Float, BigDecimal
+            [:double, argument.to_f]
+          else
+            raise ArgumentError, "unsupported MessageFormat argument type: #{argument.class}"
+          end
+        end
+      end
+
+      def validate_argument_count!(arguments)
+        unless arguments.is_a?(Array)
+          raise ArgumentError, 'MessageFormat arguments must be provided as an Array of numbered values'
+        end
+
+        highest_argument = argument_types.keys.max
+        required_count = highest_argument ? highest_argument + 1 : 0
+        return if arguments.length >= required_count
+
+        raise ArgumentError, "MessageFormat pattern requires at least #{required_count} numbered arguments"
+      end
+
+      # Maps each numbered argument in the pattern to its ICU type keyword
+      # (e.g. "plural", "select", "number") or nil for a simple argument.
+      def argument_types
+        types = {}
+        quoted = false
+        chars = @pattern.each_char.to_a
+        index = 0
+
+        while index < chars.length
+          if chars[index] == "'"
+            if chars[index + 1] == "'"
+              index += 2
+              next
+            elsif quoted
+              quoted = false
+            elsif ['{', '}', '#'].include?(chars[index + 1])
+              quoted = true
+            end
+          elsif chars[index] == '{' && !quoted
+            index += 1
+            index += 1 while index < chars.length && chars[index].match?(/\s/)
+            start = index
+            index += 1 while index < chars.length && chars[index].match?(/\d/)
+            next if index == start
+
+            arg_number = chars[start...index].join.to_i
+            index += 1 while index < chars.length && chars[index].match?(/\s/)
+
+            if chars[index] == '}'
+              types[arg_number] = nil unless types.key?(arg_number)
+            elsif chars[index] == ','
+              index += 1
+              index += 1 while index < chars.length && chars[index].match?(/\s/)
+              type_start = index
+              index += 1 while index < chars.length && chars[index].match?(/[A-Za-z]/)
+              type = chars[type_start...index].join.downcase
+              types[arg_number] = (type.empty? ? nil : type) unless types.key?(arg_number)
+            end
+          end
+          index += 1
+        end
+
+        types
+      end
+
+      def validate_argument_types!(arguments)
+        types = argument_types
+        arguments.each_with_index do |argument, index|
+          next unless types.key?(index)
+
+          case expected_kind_for(types[index])
+          when :string
+            unless argument.is_a?(String)
+              raise ArgumentError,
+                    "MessageFormat argument #{index} expects a String, got #{argument.class}"
+            end
+          when :number
+            unless [Integer, Float, BigDecimal].any? { |klass| argument.is_a?(klass) }
+              raise ArgumentError,
+                    "MessageFormat argument #{index} expects a numeric value, got #{argument.class}"
+            end
+          end
+        end
+      end
+
+      # ICU reads each argument slot from the vararg list according to the
+      # pattern: simple and select arguments read a UChar*, while number,
+      # date, time, choice, plural, and selectordinal read a double.
+      def expected_kind_for(type)
+        case type
+        when 'number', 'date', 'time', 'choice', 'plural', 'selectordinal' then :number
+        else :string
+        end
+      end
+
+      def null_terminated_uchar_pointer(string)
+        pointer = UCharPointer.from_utf8(string)
+        terminated = UCharPointer.new(pointer.length_in_uchars + 1)
+        terminated.put_bytes(0, pointer.get_bytes(0, pointer.size))
+        terminated.put_uint16(pointer.size, 0)
+        terminated
+      end
+    end
+  end
+end

data/lib/ffi-icu/normalization.rb

@@ -3,28 +3,28 @@
 module ICU
   module Normalization
     def self.normalize(input, mode = :default)
-      input_length  = input.size
       needed_length = out_length = options = 0
-      in_ptr        = UCharPointer.from_string(input)
+      input_pointer = UCharPointer.from_utf8(input)
+      input_length  = input_pointer.length_in_uchars
       out_ptr       = UCharPointer.new(out_length)
 
       retried = false
 
       begin
         Lib.check_error do |error|
-          needed_length = Lib.unorm_normalize(in_ptr, input_length, mode, options, out_ptr, out_length, error)
+          needed_length = Lib.unorm_normalize(input_pointer, input_length, mode, options, out_ptr, out_length, error)
         end
       rescue BufferOverflowError
         raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
-        out_ptr       = out_ptr.resized_to(needed_length)
         out_length    = needed_length + 1
+        out_ptr       = out_ptr.resized_to(out_length)
 
         retried = true
         retry
       end
 
-      out_ptr.string
+      out_ptr.utf8_string(needed_length)
     end
   end
 end

data/lib/ffi-icu/normalizer.rb

@@ -11,39 +11,38 @@ module ICU
     end
 
     def normalize(input)
-      input_length  = input.size
-      in_ptr        = UCharPointer.from_string(input)
+      input_pointer = UCharPointer.from_utf8(input)
+      input_length  = input_pointer.length_in_uchars
       needed_length = capacity = 0
       out_ptr       = UCharPointer.new(needed_length)
 
       retried = false
       begin
         Lib.check_error do |error|
-          needed_length = Lib.unorm2_normalize(@instance, in_ptr, input_length, out_ptr, capacity, error)
+          needed_length = Lib.unorm2_normalize(@instance, input_pointer, input_length, out_ptr, capacity, error)
         end
       rescue BufferOverflowError
         raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
-        capacity = needed_length
-        out_ptr = out_ptr.resized_to(needed_length)
+        capacity = needed_length + 1
+        out_ptr = out_ptr.resized_to(capacity)
 
         retried = true
         retry
       end
 
-      out_ptr.string
+      out_ptr.utf8_string(needed_length)
     end
 
-    def normailzed?(input)
-      input_length  = input.size
-      in_ptr        = UCharPointer.from_string(input)
+    def normalized?(input)
+      input_pointer = UCharPointer.from_utf8(input)
+      input_length  = input_pointer.length_in_uchars
 
       Lib.check_error do |error|
-        Lib.unorm2_isNormalized(@instance, in_ptr, input_length, error)
+        Lib.unorm2_isNormalized(@instance, input_pointer, input_length, error)
       end
-
-      result
     end
+    alias normailzed? normalized?
 
     def is_normalized?(input) # rubocop:disable Naming/PredicatePrefix
       Warning.warn('is_normalized? is deprecated and will be removed after v0.7. Please use normalized? instead.')

data/lib/ffi-icu/number_formatting.rb

@@ -97,7 +97,7 @@ module ICU
                               end
             end
           end
-          out_ptr.string(needed_length)
+          out_ptr.utf8_string(needed_length)
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -136,10 +136,10 @@ module ICU
 
         begin
           Lib.check_error do |error|
-            needed_length = Lib.unum_format_currency(@f, number, UCharPointer.from_string(currency, 4), out_ptr,
+            needed_length = Lib.unum_format_currency(@f, number, UCharPointer.from_utf8(currency, 4), out_ptr,
                                                      needed_length, nil, error)
           end
-          out_ptr.string
+          out_ptr.utf8_string
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 

data/lib/ffi-icu/time_formatting.rb

@@ -91,8 +91,8 @@ module ICU
         pattern_ptr = FFI::MemoryPointer.new(4)
 
         if time_zone_str
-          time_zone = UCharPointer.from_string(time_zone_str)
-          tz_len = time_zone_str.size
+          time_zone = UCharPointer.from_utf8(time_zone_str)
+          tz_len = time_zone.length_in_uchars
         else
           Lib.check_error do |error|
             i_len = 150
@@ -142,8 +142,8 @@ module ICU
       end
 
       def parse(str)
-        str_u = UCharPointer.from_string(str)
-        str_l = str.size
+        str_u = UCharPointer.from_utf8(str)
+        str_l = str_u.length_in_uchars
         Lib.check_error do |error|
           ret = Lib.udat_parse(@f, str_u, str_l, nil, error)
           Time.at(ret / 1000.0)
@@ -170,7 +170,7 @@ module ICU
             end
           end
 
-          out_ptr.string
+          out_ptr.utf8_string
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -209,7 +209,7 @@ module ICU
             needed_length = Lib.udat_toPattern(@f, localized, out_ptr, needed_length, error)
           end
 
-          out_ptr.string
+          out_ptr.utf8_string
         rescue BufferOverflowError
           raise(BufferOverflowError, "needed: #{needed_length}") if retried
 
@@ -228,8 +228,8 @@ module ICU
       end
 
       def skeleton_format(skeleton_pattern_str, locale)
-        skeleton_pattern_ptr = UCharPointer.from_string(skeleton_pattern_str)
-        skeleton_pattern_len = skeleton_pattern_str.size
+        skeleton_pattern_ptr = UCharPointer.from_utf8(skeleton_pattern_str)
+        skeleton_pattern_len = skeleton_pattern_ptr.length_in_uchars
 
         needed_length = 0
         pattern_ptr = UCharPointer.new(needed_length)
@@ -272,7 +272,7 @@ module ICU
         return unless @hour_cycle
 
         # Get the current pattern and convert to a skeleton
-        skeleton_str = pattern_to_skeleton_uchar(current_pattern_as_uchar).string
+        skeleton_str = pattern_to_skeleton_uchar(current_pattern_as_uchar).utf8_string
 
         # Manipulate the skeleton to make it work with the correct hour cycle.
         skeleton_str.gsub!(/[hHkKjJ]/, HOUR_CYCLE_SYMS[@hour_cycle])
@@ -286,7 +286,7 @@ module ICU
         end
 
         # Convert the skeleton back to a pattern
-        new_pattern_str = skeleton_to_pattern_uchar(UCharPointer.from_string(skeleton_str)).string
+        new_pattern_str = skeleton_to_pattern_uchar(UCharPointer.from_utf8(skeleton_str)).utf8_string
 
         # We also need to manipulate the _pattern_, a little bit, because (according to Firefox source):
         #
@@ -345,8 +345,8 @@ module ICU
       end
 
       def set_date_format_impl(localized, pattern_str)
-        pattern     = UCharPointer.from_string(pattern_str)
-        pattern_len = pattern_str.size
+        pattern     = UCharPointer.from_utf8(pattern_str)
+        pattern_len = pattern.length_in_uchars
 
         Lib.check_error do |_error|
           Lib.udat_applyPattern(@f, localized, pattern, pattern_len)

data/lib/ffi-icu/transliteration.rb

@@ -26,15 +26,17 @@ module ICU
         rules_length = 0
 
         if rules
-          rules_length = rules.size + 1
-          rules = UCharPointer.from_string(rules)
+          rules = UCharPointer.from_utf8(rules)
+          rules_length = rules.length_in_uchars
         end
 
+        id = UCharPointer.from_utf8(id)
+
         parse_error = Lib::UParseError.new
         begin
           Lib.check_error do |status|
-            ptr = Lib.utrans_openU(UCharPointer.from_string(id), id.size, direction, rules, rules_length,
-                                   @parse_error, status)
+            ptr = Lib.utrans_openU(id, id.length_in_uchars, direction, rules, rules_length,
+                                   parse_error, status)
             @tr = FFI::AutoPointer.new(ptr, Lib.method(:utrans_close))
           end
         rescue ICU::Error => e
@@ -45,9 +47,11 @@ module ICU
       def transliterate(from)
         # this is a bit unpleasant
 
-        unicode_size = from.unpack('U*').size
-        capacity     = unicode_size + 1
-        buf          = UCharPointer.from_string(from, capacity)
+        input = UCharPointer.from_utf8(from)
+        input_length_in_uchars = input.length_in_uchars
+        capacity               = input_length_in_uchars + 1
+        buf = UCharPointer.new(capacity)
+        buf.put_bytes(0, input.get_bytes(0, input.size))
         limit        = FFI::MemoryPointer.new(:int32)
         text_length  = FFI::MemoryPointer.new(:int32)
 
@@ -56,7 +60,7 @@ module ICU
         begin
           # resets to original size on retry
           [limit, text_length].each do |ptr|
-            ptr.put_int32(0, unicode_size)
+            ptr.put_int32(0, input_length_in_uchars)
           end
 
           Lib.check_error do |error|
@@ -73,13 +77,13 @@ module ICU
           # create a new buffer with more capacity instead of resizing,
           # since the old buffer now has result data
           buf.free
-          buf = UCharPointer.from_string(from, capacity)
+          buf = UCharPointer.from_utf8(from, capacity)
 
           retried = true
           retry
         end
 
-        buf.string(text_length.get_int32(0))
+        buf.utf8_string(text_length.get_int32(0))
       end
     end
   end

data/lib/ffi-icu/uchar.rb

@@ -22,6 +22,22 @@ module ICU
       ptr
     end
 
+    def self.from_utf8(str, capacity = nil)
+      chars = str.encode(Encoding::UTF_16LE).unpack('v*')
+
+      if capacity
+        raise(ArgumentError, "capacity is too small for string of #{chars.size} UChars") if capacity < chars.size
+
+        ptr = new(capacity)
+      else
+        ptr = new(chars.size)
+      end
+
+      ptr.write_array_of_uint16(chars)
+
+      ptr
+    end
+
     def initialize(size)
       super(UCHAR_TYPE, size)
     end
@@ -42,6 +58,13 @@ module ICU
       wstring.pack('U*')
     end
 
+    def utf8_string(length = nil)
+      length ||= size / TYPE_SIZE
+
+      wstring = read_array_of_uint16(length).pack('v*')
+      wstring.force_encoding(Encoding::UTF_16LE).encode(Encoding::UTF_8)
+    end
+
     def length_in_uchars
       size / type_size
     end

data/lib/ffi-icu/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ICU
-  VERSION = '0.6.1'
+  VERSION = '0.7.0'
 end

data/lib/ffi-icu.rb

@@ -25,11 +25,14 @@ require 'ffi-icu/lib/util'
 require 'ffi-icu/uchar'
 require 'ffi-icu/chardet'
 require 'ffi-icu/collation'
+require 'ffi-icu/currency'
 require 'ffi-icu/locale'
 require 'ffi-icu/transliteration'
 require 'ffi-icu/normalization'
 require 'ffi-icu/normalizer'
 require 'ffi-icu/break_iterator'
 require 'ffi-icu/number_formatting'
+require 'ffi-icu/message_formatting'
 require 'ffi-icu/time_formatting'
 require 'ffi-icu/duration_formatting'
+require 'ffi-icu/list_formatting'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffi-icu
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Erick Guan
@@ -76,10 +76,13 @@ files:
 - lib/ffi-icu/break_iterator.rb
 - lib/ffi-icu/chardet.rb
 - lib/ffi-icu/collation.rb
+- lib/ffi-icu/currency.rb
 - lib/ffi-icu/duration_formatting.rb
 - lib/ffi-icu/lib.rb
 - lib/ffi-icu/lib/util.rb
+- lib/ffi-icu/list_formatting.rb
 - lib/ffi-icu/locale.rb
+- lib/ffi-icu/message_formatting.rb
 - lib/ffi-icu/normalization.rb
 - lib/ffi-icu/normalizer.rb
 - lib/ffi-icu/number_formatting.rb