ffi-icu 0.1.9 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c381746aed3839388946fda0552882604cbee60c
-  data.tar.gz: 6edcf4bb0a7c7fef09c99c3cf6bf269566682736
+SHA256:
+  metadata.gz: 2066f9d1a113f07fb761d2c97f742e6cad8dde11b26c0e4d1da2ed8a85567297
+  data.tar.gz: e6df31c42fc3e518d8d19a98dec0b5aeed37ae90da3fd7570f32ef58397d531d
 SHA512:
-  metadata.gz: 7c35fdf15d4d05069c52e26e60ac8f6818775fd7c224ee8b401de582a6cc3bf29f25da78c93c8260e3398cf38349b284f417c63a10560cff479c396e999f5cd5
-  data.tar.gz: 1b1c25f761dec3bf855a5c70072f4ef3676319f1d86f8434914e0368df466e2caac85c94531f733ef0c49fbdfec8532f0c618e051d907ec75147d86a1c326e2e
+  metadata.gz: bf39178c2893939c57a8e7810d407da635ceff9b4cc5ddbe60360df53a0bf7f35b17d8f4036cf001d85edf4fa83efd8f4ed1b91d6cdb21e54d52810f536bf7ce
+  data.tar.gz: ee8c887487e3c3a0c0511e7017753b4801da3fc6a73e4d476211452026fd0ae3d8ae0d4c35be9eb5ed8827df8b757bc985dc5f5eafc77a12c872ca37347e9924

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -1,7 +1,20 @@
+language: ruby
+os: linux
+dist: focal
+
+arch:
+  - amd64
+  - arm64
+
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.0
+  - 2.6
+  - 2.7
+  - 3.0
   - ruby-head
+
 before_script:
-  - sudo apt-get install -y libicu48
+  - sudo apt-get install -y libicu-dev
+
+jobs:
+  allow_failures:
+    - arch: arm64

data/README.md

@@ -1,9 +1,7 @@
-ffi-icu
+ffi-icu [![Build Status](https://travis-ci.org/erickguan/ffi-icu.svg?branch=master)](https://travis-ci.org/erickguan/ffi-icu)
 =======
 
-Simple FFI wrappers for things I need from ICU. For the full thing, check out [ICU4R](http://icu4r.rubyforge.org/) instead.
-
-[![Build Status](https://secure.travis-ci.org/jarib/ffi-icu.png)](http://travis-ci.org/jarib/ffi-icu)
+Simple FFI wrappers for ICU. Checkout the renovated [ICU gem](https://github.com/fantasticfears/icu4r) instead which supports various of encoding and distributed with packaged source. FFI-ICU needs some love with ICU gem's transcoding method.
 
 Gem
 ---
@@ -18,7 +16,7 @@ Dependencies
 ICU.
 
 If you get messages that the library or functions are not found, you can
-set some environment varibles to tell ffi-icu where to find it, e.g.:
+set some environment variables to tell ffi-icu where to find it, e.g.:
 
     $ export FFI_ICU_LIB="icui18n.so"
     $ export FFI_ICU_VERSION_SUFFIX="_3_8"
@@ -49,7 +47,6 @@ or
 Why not just use rchardet?
 
 * speed
-* 1.9 support
 
 Locale Sensitive Collation
 --------------------------
@@ -112,14 +109,24 @@ Examples:
     f #=> "12.11.15 15:21"
 
     # reusable formatting objects
-    formater = ICU::TimeFormatting.create(:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :long , :time => :none)
-    formater.format(Time.now)  #=> "25. února 2015"
+    formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :long, :time => :none)
+    formatter.format(Time.now)  #=> "25. února 2015"
 ```
 
 ```ruby
     # reusable formatting objects
-    formater = ICU::TimeFormatting.create(:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :long , :time => :none)
-    formater.parse("25. února 2015") #=> Wed Feb 25 00:00:00 +0100 2015
+    formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :long, :time => :none)
+    formatter.parse("25. února 2015") #=> Wed Feb 25 00:00:00 +0100 2015
+```
+
+For skeleton formatting, visit the [Unicode date field symbol table](https://unicode-org.github.io/icu/userguide/format_parse/datetime/#date-field-symbol-table) page to help find the pattern characters to use.
+
+```ruby
+    formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :date => :pattern, :time => :pattern, :skeleton => 'MMMMY')
+    formatter.format(Time.now)  #=> "únor 2015"
+
+    formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :date => :pattern, :time => :pattern, :skeleton => 'Y')
+    formatter.format(Time.now)  #=> "2015"
 ```
 
 Tested on:
@@ -132,15 +139,15 @@ Platforms:
 
 Rubies:
 
-* 1.9.3
-* 2.0.0
-* 2.1.0
-* ruby-head
+- 2.5
+- 2.6
+- 2.7
+- ruby-head
 
 TODO:
 =====
 
-* Any other useful part of ICU? 
+* Any other useful part of ICU?
 * Windows?!
 
 Note on Patches/Pull Requests

data/ffi-icu.gemspec

@@ -6,7 +6,8 @@ Gem::Specification.new do |s|
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors                   = ["Jari Bakken"]
-  s.date                      = %q{2010-08-23}
+  s.date                      = %q{2019-10-15}
+  s.licenses                  = ['MIT']
   s.description               = %q{Provides charset detection, locale sensitive collation and more. Depends on libicu.}
   s.email                     = %q{jari.bakken@gmail.com}
   s.extra_rdoc_files          = ["LICENSE", "README.md"]
@@ -20,7 +21,6 @@ Gem::Specification.new do |s|
   s.summary                   = %q{Simple Ruby FFI wrappers for things I need from ICU.}
 
   s.add_runtime_dependency "ffi", "~> 1.0", ">= 1.0.9"
-  s.add_development_dependency "rspec", ["~> 2.5.0"]
-  s.add_development_dependency "rake", ["~> 0.9.2"]
+  s.add_development_dependency 'rspec', '~> 3.9'
+  s.add_development_dependency "rake", [">= 12.3.3"]
 end
-

data/lib/ffi-icu.rb

@@ -18,15 +18,6 @@ module ICU
       os
     end
   end
-
-  def self.ruby19?
-    RUBY_VERSION >= '1.9'
-  end
-end
-
-unless ICU.ruby19?
-  require 'jcode'
-  $KCODE = 'u'
 end
 
 require "ffi-icu/core_ext/string"

data/lib/ffi-icu/lib.rb

@@ -39,7 +39,12 @@ module ICU
                     [find_lib("libicui18n.#{FFI::Platform::LIBSUFFIX}.??"),
                      find_lib("libicutu.#{FFI::Platform::LIBSUFFIX}.??")]
                   when :osx
-                    [find_lib("libicucore.#{FFI::Platform::LIBSUFFIX}")]
+                    # See https://developer.apple.com/documentation/macos-release-notes/macos-big-sur-11_0_1-release-notes (62986286)
+                    if Gem::Version.new(`sw_vers -productVersion`) >= Gem::Version.new('11')
+                      ["libicucore.#{FFI::Platform::LIBSUFFIX}"]
+                    else
+                      [find_lib("libicucore.#{FFI::Platform::LIBSUFFIX}")]
+                    end
                   when :linux
                     [find_lib("libicui18n.#{FFI::Platform::LIBSUFFIX}.??"),
                      find_lib("libicutu.#{FFI::Platform::LIBSUFFIX}.??")]
@@ -99,7 +104,7 @@ module ICU
       # Here are the possible suffixes
       suffixes = [""]
       if version
-        suffixes << "_#{version}" << "_#{version[0].chr}_#{version[1].chr}"
+        suffixes << "_#{version}" << "_#{version[0].chr}_#{version[1].chr}" << "_#{version.split('.')[0]}"
       end
 
       # Try to find the u_errorName function using the possible suffixes
@@ -420,11 +425,12 @@ module ICU
     attach_function :unum_set_attribute, "unum_setAttribute#{suffix}", [:pointer, :number_format_attribute, :int32_t], :void
     # date
     enum :date_format_style, [
-      :none,  -1,
-      :full,   0,
-      :long,   1,
-      :medium, 2,
-      :short,  3,
+      :pattern, -2,
+      :none,    -1,
+      :full,    0,
+      :long,    1,
+      :medium,  2,
+      :short,   3,
     ]
     attach_function :udat_open, "udat_open#{suffix}", [:date_format_style, :date_format_style, :string, :pointer, :int32_t, :pointer, :int32_t, :pointer ], :pointer
     attach_function :udat_close, "unum_close#{suffix}", [:pointer], :void
@@ -432,6 +438,9 @@ module ICU
     attach_function :udat_parse, "udat_parse#{suffix}", [:pointer, :pointer, :int32_t,  :pointer, :pointer], :double
     attach_function :udat_toPattern, "udat_toPattern#{suffix}", [:pointer, :bool    , :pointer, :int32_t    , :pointer], :int32_t
     attach_function :udat_applyPattern, "udat_applyPattern#{suffix}", [:pointer, :bool    , :pointer, :int32_t     ], :void
+    # skeleton pattern
+    attach_function :udatpg_open, "udatpg_open#{suffix}", [:string, :pointer], :pointer
+    attach_function :udatpg_getBestPattern, "udatpg_getBestPattern#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t, :pointer], :int32_t
     # tz
     attach_function :ucal_setDefaultTimeZone, "ucal_setDefaultTimeZone#{suffix}", [:pointer, :pointer], :int32_t
     attach_function :ucal_getDefaultTimeZone, "ucal_getDefaultTimeZone#{suffix}", [:pointer, :int32_t, :pointer], :int32_t

data/lib/ffi-icu/locale.rb

@@ -75,7 +75,7 @@ module ICU
     def display_country(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(256) do |buffer, status|
         Lib.uloc_getDisplayCountry(@id, locale, buffer, buffer.size, status)
       end
     end
@@ -83,7 +83,7 @@ module ICU
     def display_language(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(192) do |buffer, status|
         Lib.uloc_getDisplayLanguage(@id, locale, buffer, buffer.size, status)
       end
     end
@@ -91,7 +91,7 @@ module ICU
     def display_name(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(256) do |buffer, status|
         Lib.uloc_getDisplayName(@id, locale, buffer, buffer.size, status)
       end
     end
@@ -99,7 +99,7 @@ module ICU
     def display_script(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(128) do |buffer, status|
         Lib.uloc_getDisplayScript(@id, locale, buffer, buffer.size, status)
       end
     end

data/lib/ffi-icu/time_formatting.rb

@@ -1,3 +1,4 @@
+require 'date'
 
 module ICU
   module TimeFormatting
@@ -64,21 +65,31 @@ module ICU
 
       private
 
-      def make_formatter(time_style, date_style, locale, time_zone_str)
-        time_zone = nil
-        d_len = 0
+      def make_formatter(time_style, date_style, locale, time_zone_str, skeleton)
+        time_zone   = nil
+        tz_len      = 0
+        pattern_len = -1
+        pattern_ptr = FFI::MemoryPointer.new(4)
+
         if time_zone_str
           time_zone = UCharPointer.from_string(time_zone_str)
-          d_len = time_zone_str.size
+          tz_len = time_zone_str.size
         else
           Lib.check_error { | error| 
             i_len  = 150
             time_zone = UCharPointer.new(i_len)
-            d_len = Lib.ucal_getDefaultTimeZone(time_zone, i_len, error) 
+            tz_len = Lib.ucal_getDefaultTimeZone(time_zone, i_len, error) 
           }
         end
 
-        ptr = Lib.check_error { | error| Lib.udat_open(time_style, date_style, locale, time_zone, d_len, FFI::MemoryPointer.new(4), -1, error) }
+        if skeleton
+          date_style = :pattern
+          time_style = :pattern
+
+          pattern_len, pattern_ptr = skeleton_format(skeleton, locale)
+        end
+
+        ptr = Lib.check_error { | error| Lib.udat_open(time_style, date_style, locale, time_zone, tz_len, pattern_ptr, pattern_len, error) }
         FFI::AutoPointer.new(ptr, Lib.method(:udat_close))
       end
     end
@@ -90,7 +101,9 @@ module ICU
         locale     = options[:locale] || 'C'
         tz_style   = options[:tz_style]
         time_zone  = options[:zone]
-        @f = make_formatter(time_style, date_style, locale, time_zone)
+        skeleton   = options[:skeleton]
+
+        @f = make_formatter(time_style, date_style, locale, time_zone, skeleton)
         if tz_style
           f0 = date_format(true)
           f1 = update_tz_format(f0, tz_style)    
@@ -176,6 +189,31 @@ module ICU
           needed_length = Lib.udat_applyPattern(@f, localized, pattern, pattern_len)
         end
       end
+
+      def skeleton_format(pattern, locale)
+          pattern = UCharPointer.from_string(pattern)
+
+          needed_length = 0
+          pattern_ptr = UCharPointer.new(needed_length)
+
+          udatpg_ptr = Lib.check_error { |error| Lib.udatpg_open(locale, error) }
+          generator = FFI::AutoPointer.new(udatpg_ptr, Lib.method(:udat_close))
+
+          retried = false
+
+        begin
+          Lib.check_error do |error|
+            needed_length = Lib.udatpg_getBestPattern(generator, pattern, pattern.size, pattern_ptr, needed_length, error)
+          end
+
+          return needed_length, pattern_ptr
+        rescue BufferOverflowError
+          raise BufferOverflowError, "needed: #{needed_length}" if retried
+          pattern_ptr = pattern_ptr.resized_to needed_length
+          retried = true
+          retry
+        end
+      end
     end # DateTimeFormatter
   end # Formatting
 end # ICU

data/lib/ffi-icu/version.rb

@@ -1,3 +1,3 @@
 module ICU
-  VERSION = "0.1.9"
+  VERSION = "0.4.0"
 end

data/spec/break_iterator_spec.rb

@@ -1,75 +1,76 @@
 # encoding: utf-8
 
-require "spec_helper"
-
 module ICU
   describe BreakIterator do
 
     it "should return available locales" do
       locales = ICU::BreakIterator.available_locales
-      locales.should be_kind_of(Array)
-      locales.should_not be_empty
-      locales.should include("en_US")
+      expect(locales).to be_an(Array)
+      expect(locales).to_not be_empty
+      expect(locales).to include("en_US")
     end
 
     it "finds all word boundaries in an English string" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."
-      iterator.to_a.should == [0, 5, 6, 11, 12, 17, 18, 21, 22, 26, 27, 28, 39, 40, 51, 52, 56, 57, 58, 61, 62, 64, 65, 72, 73, 79, 80, 90, 91, 93, 94, 100, 101, 103, 104, 110, 111, 116, 117, 123, 124]
+      expect(iterator.to_a).to eq(
+        [0, 5, 6, 11, 12, 17, 18, 21, 22, 26, 27, 28, 39, 40, 51, 52, 56, 57, 58, 61, 62, 64, 65, 72, 73, 79, 80, 90, 91, 93, 94, 100, 101, 103, 104, 110, 111, 116, 117, 123, 124]
+      )
     end
 
     it "returns each substring" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet."
 
-      iterator.substrings.should == ["Lorem", " ", "ipsum", " ", "dolor", " ", "sit", " ", "amet", "."]
+      expect(iterator.substrings).to eq(["Lorem", " ", "ipsum", " ", "dolor", " ", "sit", " ", "amet", "."])
     end
 
     it "returns the substrings of a non-ASCII string" do
       iterator = BreakIterator.new :word, "th_TH"
       iterator.text = "รู้อะไรไม่สู้รู้วิชา รู้รักษาตัวรอดเป็นยอดดี"
 
-      iterator.substrings.should ==  ["รู้", "อะไร", "ไม่สู้", "รู้", "วิชา", " ", "รู้", "รักษา", "ตัว", "รอด", "เป็น", "ยอดดี"]
+      expect(iterator.substrings).to eq(
+        ["รู้", "อะไร", "ไม่สู้", "รู้", "วิชา", " ", "รู้", "รักษา", "ตัว", "รอด", "เป็น", "ยอดดี"]
+      )
     end
 
     it "finds all word boundaries in a non-ASCII string" do
       iterator = BreakIterator.new :word, "th_TH"
       iterator.text = "การทดลอง"
-      iterator.to_a.should == [0, 3, 8]
+      expect(iterator.to_a).to eq([0, 3, 8])
     end
 
     it "finds all sentence boundaries in an English string" do
       iterator = BreakIterator.new :sentence, "en_US"
       iterator.text = "This is a sentence. This is another sentence, with a comma in it."
-      iterator.to_a.should == [0, 20, 65]
+      expect(iterator.to_a).to eq([0, 20, 65])
     end
 
     it "can navigate back and forward" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet."
 
-      iterator.first.should == 0
+      expect(iterator.first).to eq(0)
       iterator.next
-      iterator.current.should == 5
-      iterator.last.should == 27
+      expect(iterator.current).to eq(5)
+      expect(iterator.last).to eq(27)
     end
 
     it "fetches info about given offset" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet."
 
-      iterator.following(3).should == 5
-      iterator.preceding(6).should == 5
+      expect(iterator.following(3)).to eq(5)
+      expect(iterator.preceding(6)).to eq(5)
 
-      iterator.should be_boundary(5)
-      iterator.should_not be_boundary(10)
+      expect(iterator).to be_boundary(5)
+      expect(iterator).to_not be_boundary(10)
     end
 
     it "returns an Enumerator if no block was given" do
       iterator = BreakIterator.new :word, "nb"
-      expected = ICU.ruby19? ? Enumerator : Enumerable::Enumerator
 
-      iterator.each.should be_kind_of(expected)
+      expect(iterator.each).to be_kind_of(Enumerator)
     end
 
   end # BreakIterator

data/spec/chardet_spec.rb

@@ -1,29 +1,27 @@
 # encoding: UTF-8
 
-require 'spec_helper'
-
 describe ICU::CharDet::Detector do
 
   let(:detector) { ICU::CharDet::Detector.new }
 
   it "should recognize UTF-8" do
     m = detector.detect("æåø")
-    m.name.should == "UTF-8"
-    m.language.should be_kind_of(String)
+    expect(m.name).to eq("UTF-8")
+    expect(m.language).to be_a(String)
   end
 
   it "has a list of detectable charsets" do
     cs = detector.detectable_charsets
-    cs.should be_kind_of(Array)
-    cs.should_not be_empty
+    expect(cs).to be_an(Array)
+    expect(cs).to_not be_empty
 
-    cs.first.should be_kind_of(String)
+    expect(cs.first).to be_a(String)
   end
 
   it "should disable / enable the input filter" do
-    detector.input_filter_enabled?.should be_false
+    expect(detector.input_filter_enabled?).to be_falsey
     detector.input_filter_enabled = true
-    detector.input_filter_enabled?.should be_true
+    expect(detector.input_filter_enabled?).to be_truthy
   end
 
   it "should should set declared encoding" do
@@ -31,14 +29,14 @@ describe ICU::CharDet::Detector do
   end
 
   it "should detect several matching encodings" do
-    detector.detect_all("foo bar").should be_instance_of(Array)
+    expect(detector.detect_all("foo bar")).to be_an(Array)
   end
 
   it "should support null bytes" do
     # Create a utf-16 string and then force it to binary (ascii) to mimic data from net/http
     string = "foo".encode("UTF-16").force_encoding("binary")
     m = detector.detect(string)
-    m.name.should == "UTF-16BE"
-    m.language.should be_kind_of(String)
+    expect(m.name).to eq("UTF-16BE")
+    expect(m.language).to be_a(String)
   end
 end