ffi-icu 0.4.0 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2066f9d1a113f07fb761d2c97f742e6cad8dde11b26c0e4d1da2ed8a85567297
-  data.tar.gz: e6df31c42fc3e518d8d19a98dec0b5aeed37ae90da3fd7570f32ef58397d531d
+  metadata.gz: e05cacd8501af573e102abe908556c1ea0f2eb1990b560271890c9d0827feb09
+  data.tar.gz: 4c5433c4183b376548f20ff533311d1acc61035d395b63a35f8e89a387ca7d98
 SHA512:
-  metadata.gz: bf39178c2893939c57a8e7810d407da635ceff9b4cc5ddbe60360df53a0bf7f35b17d8f4036cf001d85edf4fa83efd8f4ed1b91d6cdb21e54d52810f536bf7ce
-  data.tar.gz: ee8c887487e3c3a0c0511e7017753b4801da3fc6a73e4d476211452026fd0ae3d8ae0d4c35be9eb5ed8827df8b757bc985dc5f5eafc77a12c872ca37347e9924
+  metadata.gz: a40fa82617a2c16aa759b263e5153e4135b5844823666ac5d0b5d205b300819f3ac1cf10d5e8e9f9182c2357080e6ed6add40388287b308ef6eae645f052a38f
+  data.tar.gz: 4d34feb23961b844cc9701d09a52a806f375bc58b10a04622d2230d57a1a97cd015f004ed72239a039041e901d304deb4051f81400191981116b7b055ef8b15b

data/lib/ffi-icu/lib/util.rb

@@ -28,7 +28,19 @@ module ICU
         result.read_string(length)
       end
 
-      def self.read_uchar_buffer(length)
+      def self.read_uchar_buffer(length, &blk)
+        buf, len = read_uchar_buffer_as_ptr_impl(length, &blk)
+        buf.string(len)
+      end
+
+      def self.read_uchar_buffer_as_ptr(length, &blk)
+        buf, _ = read_uchar_buffer_as_ptr_impl(length, &blk)
+        buf
+      end
+
+      private
+
+      def self.read_uchar_buffer_as_ptr_impl(length)
         attempts = 0
 
         begin
@@ -40,7 +52,7 @@ module ICU
           raise BufferOverflowError, "needed: #{length}"
         end
 
-        result.string(length)
+        [result, length]
       end
     end
   end

data/lib/ffi-icu/lib.rb

@@ -19,9 +19,7 @@ module ICU
             '/usr/local/{lib64,lib}',
             '/opt/local/{lib64,lib}',
             '/usr/{lib64,lib}',
-            '/usr/lib/x86_64-linux-gnu', # for Debian Multiarch http://wiki.debian.org/Multiarch
-            '/usr/lib/i386-linux-gnu',   # for Debian Multiarch
-          ]
+          ] + Dir['/usr/lib/*-linux-gnu'] # for Debian Multiarch http://wiki.debian.org/Multiarch
         end
       end
     end
@@ -39,8 +37,12 @@ module ICU
                     [find_lib("libicui18n.#{FFI::Platform::LIBSUFFIX}.??"),
                      find_lib("libicutu.#{FFI::Platform::LIBSUFFIX}.??")]
                   when :osx
-                    # See https://developer.apple.com/documentation/macos-release-notes/macos-big-sur-11_0_1-release-notes (62986286)
-                    if Gem::Version.new(`sw_vers -productVersion`) >= Gem::Version.new('11')
+                    if ENV.key?('FFI_ICU_LIB')
+                      # Ensure we look in the user-supplied override dir for a user-compiled libicu
+                      [find_lib("libicui18n.??.#{FFI::Platform::LIBSUFFIX}"),
+                      find_lib("libicutu.??.#{FFI::Platform::LIBSUFFIX}")]
+                    elsif Gem::Version.new(`sw_vers -productVersion`) >= Gem::Version.new('11')
+                      # See https://developer.apple.com/documentation/macos-release-notes/macos-big-sur-11_0_1-release-notes (62986286)
                       ["libicucore.#{FFI::Platform::LIBSUFFIX}"]
                     else
                       [find_lib("libicucore.#{FFI::Platform::LIBSUFFIX}")]
@@ -432,6 +434,10 @@ module ICU
       :medium,  2,
       :short,   3,
     ]
+    enum :uloc_data_locale_type, [
+      :actual_locale, 0,
+      :valid_locale, 1,
+    ]
     attach_function :udat_open, "udat_open#{suffix}", [:date_format_style, :date_format_style, :string, :pointer, :int32_t, :pointer, :int32_t, :pointer ], :pointer
     attach_function :udat_close, "unum_close#{suffix}", [:pointer], :void
     attach_function :udat_format, "udat_format#{suffix}", [:pointer, :double, :pointer, :int32_t, :pointer, :pointer], :int32_t
@@ -440,7 +446,9 @@ module ICU
     attach_function :udat_applyPattern, "udat_applyPattern#{suffix}", [:pointer, :bool    , :pointer, :int32_t     ], :void
     # skeleton pattern
     attach_function :udatpg_open, "udatpg_open#{suffix}", [:string, :pointer], :pointer
+    attach_function :udatpg_close, "udatpg_close#{suffix}", [:pointer], :void
     attach_function :udatpg_getBestPattern, "udatpg_getBestPattern#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t, :pointer], :int32_t
+    attach_function :udatpg_getSkeleton, "udatpg_getSkeleton#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t, :pointer], :int32_t
     # tz
     attach_function :ucal_setDefaultTimeZone, "ucal_setDefaultTimeZone#{suffix}", [:pointer, :pointer], :int32_t
     attach_function :ucal_getDefaultTimeZone, "ucal_getDefaultTimeZone#{suffix}", [:pointer, :int32_t, :pointer], :int32_t

data/lib/ffi-icu/number_formatting.rb

@@ -68,8 +68,19 @@ module ICU
             case number
             when Float
               needed_length = Lib.unum_format_double(@f, number, out_ptr, needed_length, nil, error)
-            when Fixnum
-              needed_length = Lib.unum_format_int32(@f, number, out_ptr, needed_length, nil, error)
+            when Integer
+              begin
+                # Try doing it fast, for integers that can be marshaled into an int64_t
+                needed_length = Lib.unum_format_int64(@f, number, out_ptr, needed_length, nil, error)
+              rescue RangeError
+                # Fall back to stringifying in Ruby and passing that to ICU
+                unless defined? Lib.unum_format_decimal
+                  raise RangeError,"Number #{number} is too big to fit in int64_t and your "\
+                                    "ICU version is too old to have unum_format_decimal"
+                end
+                string_version = number.to_s
+                needed_length = Lib.unum_format_decimal(@f, string_version, string_version.bytesize, out_ptr, needed_length, nil, error)
+              end
             when BigDecimal
               string_version = number.to_s('F')
               if Lib.respond_to? :unum_format_decimal
@@ -77,8 +88,6 @@ module ICU
               else
                 needed_length = Lib.unum_format_double(@f, number.to_f, out_ptr, needed_length, nil, error)
               end
-            when Bignum
-              needed_length = Lib.unum_format_int64(@f, number, out_ptr, needed_length, nil, error)
             end
           end
           out_ptr.string needed_length

data/lib/ffi-icu/time_formatting.rb

@@ -38,6 +38,14 @@ module ICU
                                     #   (for example, "Unknown City"), such as Los Angeles
       # see: http://unicode.org/reports/tr35/tr35-dates.html#Date_Format_Patterns
     }
+
+    HOUR_CYCLE_SYMS = {
+      'h11' => 'K',
+      'h12' => 'h',
+      'h23' => 'H',
+      'h24' => 'k',
+      :locale => 'j',
+    }
     @default_options = {}
     
     def self.create(options = {})
@@ -96,14 +104,19 @@ module ICU
 
     class DateTimeFormatter < BaseFormatter
       def initialize(options={})
-        time_style = options[:time]   || :short
-        date_style = options[:date]   || :short
-        locale     = options[:locale] || 'C'
-        tz_style   = options[:tz_style]
-        time_zone  = options[:zone]
-        skeleton   = options[:skeleton]
-
-        @f = make_formatter(time_style, date_style, locale, time_zone, skeleton)
+        time_style  = options[:time]   || :short
+        date_style  = options[:date]   || :short
+        @locale     = options[:locale] || 'C'
+        tz_style    = options[:tz_style]
+        time_zone   = options[:zone]
+        skeleton    = options[:skeleton]
+        @hour_cycle = options[:hour_cycle]
+
+        if @hour_cycle && !HOUR_CYCLE_SYMS.keys.include?(@hour_cycle)
+          raise ICU::Error.new("Unknown hour cycle #{@hour_cycle}")
+        end
+
+        @f = make_formatter(time_style, date_style, @locale, time_zone, skeleton)
         if tz_style
           f0 = date_format(true)
           f1 = update_tz_format(f0, tz_style)    
@@ -111,11 +124,13 @@ module ICU
             set_date_format(true, f1)
           end
         end
+
+        replace_hour_symbol!
       end
 
       def parse(str)
           str_u = UCharPointer.from_string(str)
-          str_l = str_u.size
+          str_l = str.size
           Lib.check_error do |error|
             ret = Lib.udat_parse(@f, str_u, str_l, nil, error)
             Time.at(ret / 1000.0)
@@ -182,28 +197,28 @@ module ICU
       end
 
       def set_date_format(localized, pattern_str)
-        pattern     = UCharPointer.from_string(pattern_str)
-        pattern_len = pattern_str.size
+        set_date_format_impl(localized, pattern_str)
 
-        Lib.check_error do |error|
-          needed_length = Lib.udat_applyPattern(@f, localized, pattern, pattern_len)
-        end
+        # After setting the date format string, we need to ensure that any hour
+        # symbols were properly localised according to @hour_cycle.
+        replace_hour_symbol!
       end
 
-      def skeleton_format(pattern, locale)
-          pattern = UCharPointer.from_string(pattern)
+      def skeleton_format(skeleton_pattern_str, locale)
+          skeleton_pattern_ptr = UCharPointer.from_string(skeleton_pattern_str)
+          skeleton_pattern_len = skeleton_pattern_str.size
 
           needed_length = 0
           pattern_ptr = UCharPointer.new(needed_length)
 
           udatpg_ptr = Lib.check_error { |error| Lib.udatpg_open(locale, error) }
-          generator = FFI::AutoPointer.new(udatpg_ptr, Lib.method(:udat_close))
+          generator = FFI::AutoPointer.new(udatpg_ptr, Lib.method(:udatpg_close))
 
           retried = false
 
         begin
           Lib.check_error do |error|
-            needed_length = Lib.udatpg_getBestPattern(generator, pattern, pattern.size, pattern_ptr, needed_length, error)
+            needed_length = Lib.udatpg_getBestPattern(generator, skeleton_pattern_ptr, skeleton_pattern_len, pattern_ptr, needed_length, error)
           end
 
           return needed_length, pattern_ptr
@@ -214,6 +229,106 @@ module ICU
           retry
         end
       end
+
+      private
+
+      # Converts the current pattern to a pattern that takes the desired hour cycle
+      # into account. This is needed because most of the standard patterns in ICU
+      # contain either h (12 hour) or H (23 hour) in them, instead of j (locale-
+      # specified hour cycle). This means if you use a locale with an @hours=h12
+      # keyword in it, for example, it would normally be totally ignored by ICU.
+      #
+      # This is the same fixup done by Firefox:
+      # https://github.com/tc39/ecma402/issues/665#issuecomment-1084833809
+      # https://searchfox.org/mozilla-central/rev/625c3d0c8ae46502aed83f33bd530cb93e926e9f/intl/components/src/DateTimeFormat.cpp#282-323
+      def replace_hour_symbol!
+        # Short circuit this case - nil means "use whatever is in the pattern already", so
+        # no need to actually run any of this implementation.
+        return unless @hour_cycle
+
+        # Get the current pattern and convert to a skeleton
+        skeleton_str = pattern_to_skeleton_uchar(current_pattern_as_uchar).string
+
+        # Manipulate the skeleton to make it work with the correct hour cycle.
+        skeleton_str.gsub!(/[hHkKjJ]/, HOUR_CYCLE_SYMS[@hour_cycle])
+
+        # Either ensure the skeleton has, or does not have, am/pm, as appropriate
+        if ['h11', 'h12'].include?(@hour_cycle)
+          # Only actually append 'am/pm' if there is an hour in the format string
+          if skeleton_str =~ /[hHkKjJ]/ && !skeleton_str.include?('a')
+            skeleton_str << 'a'
+          end
+        else
+          skeleton_str.gsub!('a', '')
+        end
+
+        # Convert the skeleton back to a pattern
+        new_pattern_str = skeleton_to_pattern_uchar(UCharPointer.from_string(skeleton_str)).string
+
+        # We also need to manipulate the _pattern_, a little bit, because (according to Firefox source):
+        #
+        #     Input skeletons don't differentiate between "K" and "h" resp. "k" and "H".
+        #
+        # https://searchfox.org/mozilla-central/rev/625c3d0c8ae46502aed83f33bd530cb93e926e9f/intl/components/src/DateTimeFormat.cpp#183
+        # So, if we put a skeleton with a k in it into getBestPattern, it comes out with a H (and a
+        # skeleton with a K in it comes out with a h). Need to fix this in the generated pattern.
+        resolved_hour_cycle = @hour_cycle == :locale ? Locale.new(@locale).keyword('hours') : @hour_cycle
+
+        if HOUR_CYCLE_SYMS.keys.include?(resolved_hour_cycle)
+          new_pattern_str.gsub!(/[hHkK]/, HOUR_CYCLE_SYMS[resolved_hour_cycle])
+        end
+
+        # Finally, set the new pattern onto the date time formatter
+        set_date_format_impl(false, new_pattern_str)
+      end
+
+      # Load up the date formatter locale and make a generator
+      # Note that we _MUST_ actually use @locale as passed to us, rather than calling
+      # udat_getLocaleByType to look it up from @f, because the latter will throw away
+      # any @hours specifier in the locale, and we need it.
+      def datetime_pattern_generator
+        @datetime_pattern_generator ||= FFI::AutoPointer.new(
+          Lib.check_error { |error| Lib.udatpg_open(@locale, error) },
+          Lib.method(:udatpg_close)
+        )
+      end
+
+      def current_pattern_as_uchar
+        Lib::Util.read_uchar_buffer_as_ptr(0) do |buf, error|
+          Lib.udat_toPattern(@f, false, buf, buf.length_in_uchars, error)
+        end
+      end
+
+      def pattern_to_skeleton_uchar(pattern_uchar)
+        Lib::Util.read_uchar_buffer_as_ptr(0) do |buf, error|
+          Lib.udatpg_getSkeleton(
+            datetime_pattern_generator,
+            pattern_uchar, pattern_uchar.length_in_uchars,
+            buf, buf.length_in_uchars,
+            error
+          )
+        end
+      end
+
+      def skeleton_to_pattern_uchar(skeleton_uchar)
+        Lib::Util.read_uchar_buffer_as_ptr(0) do |buf, error|
+          Lib.udatpg_getBestPattern(
+            datetime_pattern_generator,
+            skeleton_uchar, skeleton_uchar.length_in_uchars,
+            buf, buf.length_in_uchars,
+            error
+          )
+        end
+      end
+
+      def set_date_format_impl(localized, pattern_str)
+        pattern     = UCharPointer.from_string(pattern_str)
+        pattern_len = pattern_str.size
+
+        Lib.check_error do |error|
+          needed_length = Lib.udat_applyPattern(@f, localized, pattern, pattern_len)
+        end
+      end
     end # DateTimeFormatter
   end # Formatting
 end # ICU

data/lib/ffi-icu/uchar.rb

@@ -43,6 +43,10 @@ module ICU
       wstring.pack("U*")
     end
 
+    def length_in_uchars
+      size / type_size
+    end
+
 
   end # UCharPointer
 end # ICU

data/lib/ffi-icu/version.rb

@@ -1,3 +1,3 @@
 module ICU
-  VERSION = "0.4.0"
+  VERSION = "0.4.3"
 end

data/spec/number_formatting_spec.rb

@@ -69,6 +69,11 @@ module ICU
           expect { NumberFormatting.create('en-US', :currency, style: :iso) }.to raise_error(StandardError)
         end
       end
+
+      it 'should format a bignum' do
+        str = NumberFormatting.format_number("en", 1_000_000_000_000_000_000_000_000_000_000_000_000_000)
+        expect(str).to eq('1,000,000,000,000,000,000,000,000,000,000,000,000,000')
+      end
     end
   end # NumberFormatting
 end # ICU

data/spec/time_spec.rb

@@ -95,6 +95,98 @@ module ICU
           expect(f4.date_format(true)).to eq("MMM y")
         end
       end
+
+      context 'hour cycle' do
+        # en_AU normally is 12 hours, fr_FR is normally 23 hours
+        ['en_AU', 'fr_FR', 'zh_CN'].each do |locale_name|
+          context "with locale #{locale_name}" do
+            it 'works with hour_cycle: h11' do
+              t = Time.new(2021, 04, 01, 12, 05, 0, "+00:00")
+              str = TimeFormatting.format(t, time: :short, date: :none, locale: locale_name, zone: 'UTC', hour_cycle: 'h11')
+              expect(str).to match(/0:05/i)
+              expect(str).to match(/(pm|下午)/i)
+            end
+
+            it 'works with hour_cycle: h12' do
+              t = Time.new(2021, 04, 01, 12, 05, 0, "+00:00")
+              str = TimeFormatting.format(t, time: :short, date: :none, locale: locale_name, zone: 'UTC', hour_cycle: 'h12')
+              expect(str).to match(/12:05/i)
+              expect(str).to match(/(pm|下午)/i)
+            end
+
+            it 'works with hour_cycle: h23' do
+              t = Time.new(2021, 04, 01, 00, 05, 0, "+00:00")
+              str = TimeFormatting.format(t, time: :short, date: :none, locale: locale_name, zone: 'UTC', hour_cycle: 'h23')
+              expect(str).to match(/0:05/i)
+              expect(str).to_not match(/(am|pm)/i)
+            end
+
+            it 'works with hour_cycle: h24' do
+              t = Time.new(2021, 04, 01, 00, 05, 0, "+00:00")
+              str = TimeFormatting.format(t, time: :short, date: :none, locale: locale_name, zone: 'UTC', hour_cycle: 'h24')
+              expect(str).to match(/24:05/i)
+              expect(str).to_not match(/(am|pm)/i)
+            end
+
+            it 'does not include am/pm if time is not requested' do
+              t = Time.new(2021, 04, 01, 00, 05, 0, "+00:00")
+              str = TimeFormatting.format(t, time: :none, date: :short, locale: locale_name, zone: 'UTC', hour_cycle: 'h12')
+              expect(str).to_not match(/(am|pm|下午|上午)/i)
+            end
+
+            context '@hours keyword' do
+              before(:each) do
+                skip("Only works on ICU >= 67") if Lib.version.to_a[0] < 67
+              end
+
+              it 'works with @hours=h11 keyword' do
+                t = Time.new(2021, 04, 01, 12, 05, 0, "+00:00")
+                locale = Locale.new(locale_name).with_keyword('hours', 'h11').to_s
+                str = TimeFormatting.format(t, time: :short, date: :none, locale: locale, zone: 'UTC', hour_cycle: :locale)
+                expect(str).to match(/0:05/i)
+                expect(str).to match(/(pm|下午)/i)
+              end
+              it 'works with @hours=h12 keyword' do
+                t = Time.new(2021, 04, 01, 12, 05, 0, "+00:00")
+                locale = Locale.new(locale_name).with_keyword('hours', 'h12').to_s
+                str = TimeFormatting.format(t, time: :short, date: :none, locale: locale, zone: 'UTC', hour_cycle: :locale)
+                expect(str).to match(/12:05/i)
+                expect(str).to match(/(pm|下午)/i)
+              end
+
+              it 'works with @hours=h23 keyword' do
+                t = Time.new(2021, 04, 01, 00, 05, 0, "+00:00")
+                locale = Locale.new(locale_name).with_keyword('hours', 'h23').to_s
+                str = TimeFormatting.format(t, time: :short, date: :none, locale: locale, zone: 'UTC', hour_cycle: :locale)
+                expect(str).to match(/0:05/i)
+                expect(str).to_not match(/(am|pm)/i)
+              end
+
+              it 'works with @hours=h24 keyword' do
+                t = Time.new(2021, 04, 01, 00, 05, 0, "+00:00")
+                locale = Locale.new(locale_name).with_keyword('hours', 'h24').to_s
+                str = TimeFormatting.format(t, time: :short, date: :none, locale: locale, zone: 'UTC', hour_cycle: :locale)
+                expect(str).to match(/24:05/i)
+                expect(str).to_not match(/(am|pm)/i)
+              end
+            end
+          end
+        end
+
+        it 'works with defaults on a h12 locale' do
+          t = Time.new(2021, 04, 01, 13, 05, 0, "+00:00")
+          str = TimeFormatting.format(t, time: :short, date: :none, locale: 'en_AU', zone: 'UTC', hour_cycle: :locale)
+          expect(str).to match(/1:05/i)
+          expect(str).to match(/pm/i)
+        end
+
+        it 'works with defaults on a h23 locale' do
+          t = Time.new(2021, 04, 01, 0, 05, 0, "+00:00")
+          str = TimeFormatting.format(t, time: :short, date: :none, locale: 'fr_FR', zone: 'UTC', hour_cycle: :locale)
+          expect(str).to match(/0:05/i)
+          expect(str).to_not match(/(am|pm)/i)
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffi-icu
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.3
 platform: ruby
 authors:
 - Jari Bakken