ffi-icu 0.1.7 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 61818359548264beab4f803a6d4b6873eb9796f9
-  data.tar.gz: 5cffff3be406e40154bc45c51b2e199d3b36eb9c
+SHA256:
+  metadata.gz: 3e1817bf83d85197ea62937c274ff98c71984ef4a432a6f76fc3cca764bf45dc
+  data.tar.gz: 2a21f2177dc151831fe89782be8cfe53d2f13a6d1038961eb1efe5c3970a27a0
 SHA512:
-  metadata.gz: 4f212ef09319ea71ea83493578d659c8e44d8be59a5d9afeb72f14bfd08ca834e00026cf397eba9ebb8737011455bc4c717fd66b3684c00e6515409d80560ccb
-  data.tar.gz: 0ab9f362d3d94351fc720f95d90ab3718a299e0061a6fec80a2ae690d2a8d44556a8ef73d22d25b50f2d684c1a9dc2bd625030b9c05f2548406c48bde810b5ae
+  metadata.gz: 6fb2d659678226632c485ea24de6e79f7d20ccf7b80b03833f680bb295ae07736957f3a9037ac1d3639b432f6974c279b580767d08315ad005c57cc6be79ed8b
+  data.tar.gz: 9c49ddef23ae27f88d4eaffbaf0cf42b670e7f07939b7c19b6255a4ac157faae25e8eb5bc973764e767b3b32d40226c4b1cc95772c629fbe4b678d4cff5ebccb

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -1,7 +1,13 @@
+language: ruby
+os: linux
+dist: xenial
+
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.0
+  - 2.5
+  - 2.6
+  - 2.7
   - ruby-head
+
 before_script:
-  - sudo apt-get install -y libicu48
+  - sudo apt-get install -y libicu-dev
+

data/README.md

@@ -1,9 +1,7 @@
-ffi-icu
+ffi-icu [![Build Status](https://travis-ci.org/erickguan/ffi-icu.svg?branch=master)](https://travis-ci.org/erickguan/ffi-icu)
 =======
 
-Simple FFI wrappers for things I need from ICU. For the full thing, check out [ICU4R](http://icu4r.rubyforge.org/) instead.
-
-[![Build Status](https://secure.travis-ci.org/jarib/ffi-icu.png)](http://travis-ci.org/jarib/ffi-icu)
+Simple FFI wrappers for ICU. Checkout the renovated [ICU gem](https://github.com/fantasticfears/icu4r) instead which supports various of encoding and distributed with packaged source. FFI-ICU needs some love with ICU gem's transcoding method.
 
 Gem
 ---
@@ -49,7 +47,6 @@ or
 Why not just use rchardet?
 
 * speed
-* 1.9 support
 
 Locale Sensitive Collation
 --------------------------
@@ -127,19 +124,20 @@ Tested on:
 
 Platforms:
 
-* OS X 10.6
-* Arch Linux
+* OS X 10.6 - 10.10
+* Travis' Linux
 
 Rubies:
 
-* MRI 1.9.1
-* MRI 1.8.7
+- 2.5
+- 2.6
+- 2.7
+- ruby-head
 
 TODO:
 =====
 
-* Useful ICU stuff:
-  - date formatting
+* Any other useful part of ICU?
 * Windows?!
 
 Note on Patches/Pull Requests

data/ffi-icu.gemspec

@@ -6,7 +6,8 @@ Gem::Specification.new do |s|
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors                   = ["Jari Bakken"]
-  s.date                      = %q{2010-08-23}
+  s.date                      = %q{2019-10-15}
+  s.licenses                  = ['MIT']
   s.description               = %q{Provides charset detection, locale sensitive collation and more. Depends on libicu.}
   s.email                     = %q{jari.bakken@gmail.com}
   s.extra_rdoc_files          = ["LICENSE", "README.md"]
@@ -20,7 +21,6 @@ Gem::Specification.new do |s|
   s.summary                   = %q{Simple Ruby FFI wrappers for things I need from ICU.}
 
   s.add_runtime_dependency "ffi", "~> 1.0", ">= 1.0.9"
-  s.add_development_dependency "rspec", ["~> 2.5.0"]
-  s.add_development_dependency "rake", ["~> 0.9.2"]
+  s.add_development_dependency 'rspec', '~> 3.9'
+  s.add_development_dependency "rake", [">= 12.3.3"]
 end
-

data/lib/ffi-icu.rb

@@ -18,15 +18,6 @@ module ICU
       os
     end
   end
-
-  def self.ruby19?
-    RUBY_VERSION >= '1.9'
-  end
-end
-
-unless ICU.ruby19?
-  require 'jcode'
-  $KCODE = 'u'
 end
 
 require "ffi-icu/core_ext/string"
@@ -38,7 +29,7 @@ require "ffi-icu/collation"
 require "ffi-icu/locale"
 require "ffi-icu/transliteration"
 require "ffi-icu/normalization"
+require "ffi-icu/normalizer"
 require "ffi-icu/break_iterator"
 require "ffi-icu/number_formatting"
 require "ffi-icu/time_formatting"
-

data/lib/ffi-icu/lib.rb

@@ -39,7 +39,12 @@ module ICU
                     [find_lib("libicui18n.#{FFI::Platform::LIBSUFFIX}.??"),
                      find_lib("libicutu.#{FFI::Platform::LIBSUFFIX}.??")]
                   when :osx
-                    [find_lib("libicucore.#{FFI::Platform::LIBSUFFIX}")]
+                    # See https://developer.apple.com/documentation/macos-release-notes/macos-big-sur-11_0_1-release-notes (62986286)
+                    if Gem::Version.new(`sw_vers -productVersion`) >= Gem::Version.new('11')
+                      ["libicucore.#{FFI::Platform::LIBSUFFIX}"]
+                    else
+                      [find_lib("libicucore.#{FFI::Platform::LIBSUFFIX}")]
+                    end
                   when :linux
                     [find_lib("libicui18n.#{FFI::Platform::LIBSUFFIX}.??"),
                      find_lib("libicutu.#{FFI::Platform::LIBSUFFIX}.??")]
@@ -99,7 +104,7 @@ module ICU
       # Here are the possible suffixes
       suffixes = [""]
       if version
-        suffixes << "_#{version}" << "_#{version[0].chr}_#{version[1].chr}"
+        suffixes << "_#{version}" << "_#{version[0].chr}_#{version[1].chr}" << "_#{version.split('.')[0]}"
       end
 
       # Try to find the u_errorName function using the possible suffixes
@@ -339,6 +344,15 @@ module ICU
 
     attach_function :unorm_normalize, "unorm_normalize#{suffix}", [:pointer, :int32_t, :normalization_mode, :int32_t, :pointer, :int32_t, :pointer], :int32_t
 
+    # http://icu-project.org/apiref/icu4c/unorm2_8h.html
+
+    if Gem::Version.new('4.4') <= Gem::Version.new(self.version)
+      enum :normalization2_mode, [ :compose, :decompose, :fcd, :compose_contiguous ]
+      attach_function :unorm2_getInstance, "unorm2_getInstance#{suffix}", [:pointer, :pointer, :normalization2_mode, :pointer], :pointer
+      attach_function :unorm2_normalize, "unorm2_normalize#{suffix}", [:pointer, :pointer, :int32_t, :pointer, :int32_t, :pointer], :int32_t
+      attach_function :unorm2_isNormalized, "unorm2_isNormalized#{suffix}", [:pointer, :pointer, :int32_t, :pointer], :bool
+    end
+
     #
     # Text Boundary Analysis
     #
@@ -392,10 +406,10 @@ module ICU
       :ignore
     ]
     enum :number_format_attribute, [
-       :parse_int_only, :grouping_used, :decimal_always_show, :max_integer_digits, 
-       :min_integer_digits, :integer_digits, :max_fraction_digits, :min_fraction_digits, 
-       :fraction_digits, :multiplier, :grouping_size, :rounding_mode, 
-       :rounding_increment, :format_width, :padding_position, :secondary_grouping_size, 
+       :parse_int_only, :grouping_used, :decimal_always_show, :max_integer_digits,
+       :min_integer_digits, :integer_digits, :max_fraction_digits, :min_fraction_digits,
+       :fraction_digits, :multiplier, :grouping_size, :rounding_mode,
+       :rounding_increment, :format_width, :padding_position, :secondary_grouping_size,
        :significant_digits_used, :min_significant_digits, :max_significant_digits, :lenient_parse
     ]
     attach_function :unum_open, "unum_open#{suffix}", [:number_format_style, :pointer, :int32_t, :string, :pointer, :pointer ], :pointer

data/lib/ffi-icu/locale.rb

@@ -75,7 +75,7 @@ module ICU
     def display_country(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(256) do |buffer, status|
         Lib.uloc_getDisplayCountry(@id, locale, buffer, buffer.size, status)
       end
     end
@@ -83,7 +83,7 @@ module ICU
     def display_language(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(192) do |buffer, status|
         Lib.uloc_getDisplayLanguage(@id, locale, buffer, buffer.size, status)
       end
     end
@@ -91,7 +91,7 @@ module ICU
     def display_name(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(256) do |buffer, status|
         Lib.uloc_getDisplayName(@id, locale, buffer, buffer.size, status)
       end
     end
@@ -99,7 +99,7 @@ module ICU
     def display_script(locale = nil)
       locale = locale.to_s unless locale.nil?
 
-      Lib::Util.read_uchar_buffer(64) do |buffer, status|
+      Lib::Util.read_uchar_buffer(128) do |buffer, status|
         Lib.uloc_getDisplayScript(@id, locale, buffer, buffer.size, status)
       end
     end

data/lib/ffi-icu/normalizer.rb

@@ -0,0 +1,47 @@
+module ICU
+  class Normalizer
+    # support for newer ICU normalization API
+
+    def initialize(package_name = nil, name = 'nfc', mode = :decompose)
+      Lib.check_error do |error|
+        @instance = Lib.unorm2_getInstance(package_name, name, mode, error)
+      end
+    end
+
+    def normalize(input)
+      input_length  = input.jlength
+      in_ptr        = UCharPointer.from_string(input)
+      needed_length = capacity = 0
+      out_ptr       = UCharPointer.new(needed_length)
+
+      retried = false
+      begin
+        Lib.check_error do |error|
+          needed_length = Lib.unorm2_normalize(@instance, in_ptr, input_length, out_ptr, capacity, error)
+        end
+      rescue BufferOverflowError
+        raise BufferOverflowError, "needed: #{needed_length}" if retried
+
+        capacity = needed_length
+        out_ptr = out_ptr.resized_to needed_length
+
+        retried = true
+        retry
+      end
+
+      out_ptr.string
+    end
+
+    def is_normailzed?(input)
+      input_length  = input.jlength
+      in_ptr        = UCharPointer.from_string(input)
+
+      Lib.check_error do |error|
+        result = Lib.unorm2_isNormalized(@instance, in_ptr, input_length, error)
+      end
+
+      result
+    end
+
+  end # Normalizer
+end # ICU

data/lib/ffi-icu/time_formatting.rb

@@ -1,3 +1,4 @@
+require 'date'
 
 module ICU
   module TimeFormatting

data/lib/ffi-icu/version.rb

@@ -1,3 +1,3 @@
 module ICU
-  VERSION = "0.1.7"
+  VERSION = "0.3.0"
 end

data/spec/break_iterator_spec.rb

@@ -1,75 +1,76 @@
 # encoding: utf-8
 
-require "spec_helper"
-
 module ICU
   describe BreakIterator do
 
     it "should return available locales" do
       locales = ICU::BreakIterator.available_locales
-      locales.should be_kind_of(Array)
-      locales.should_not be_empty
-      locales.should include("en_US")
+      expect(locales).to be_an(Array)
+      expect(locales).to_not be_empty
+      expect(locales).to include("en_US")
     end
 
     it "finds all word boundaries in an English string" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."
-      iterator.to_a.should == [0, 5, 6, 11, 12, 17, 18, 21, 22, 26, 27, 28, 39, 40, 51, 52, 56, 57, 58, 61, 62, 64, 65, 72, 73, 79, 80, 90, 91, 93, 94, 100, 101, 103, 104, 110, 111, 116, 117, 123, 124]
+      expect(iterator.to_a).to eq(
+        [0, 5, 6, 11, 12, 17, 18, 21, 22, 26, 27, 28, 39, 40, 51, 52, 56, 57, 58, 61, 62, 64, 65, 72, 73, 79, 80, 90, 91, 93, 94, 100, 101, 103, 104, 110, 111, 116, 117, 123, 124]
+      )
     end
 
     it "returns each substring" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet."
 
-      iterator.substrings.should == ["Lorem", " ", "ipsum", " ", "dolor", " ", "sit", " ", "amet", "."]
+      expect(iterator.substrings).to eq(["Lorem", " ", "ipsum", " ", "dolor", " ", "sit", " ", "amet", "."])
     end
 
     it "returns the substrings of a non-ASCII string" do
       iterator = BreakIterator.new :word, "th_TH"
       iterator.text = "รู้อะไรไม่สู้รู้วิชา รู้รักษาตัวรอดเป็นยอดดี"
 
-      iterator.substrings.should ==  ["รู้", "อะไร", "ไม่สู้", "รู้", "วิชา", " ", "รู้", "รักษา", "ตัว", "รอด", "เป็น", "ยอดดี"]
+      expect(iterator.substrings).to eq(
+        ["รู้", "อะไร", "ไม่สู้", "รู้", "วิชา", " ", "รู้", "รักษา", "ตัว", "รอด", "เป็น", "ยอดดี"]
+      )
     end
 
     it "finds all word boundaries in a non-ASCII string" do
       iterator = BreakIterator.new :word, "th_TH"
       iterator.text = "การทดลอง"
-      iterator.to_a.should == [0, 3, 8]
+      expect(iterator.to_a).to eq([0, 3, 8])
     end
 
     it "finds all sentence boundaries in an English string" do
       iterator = BreakIterator.new :sentence, "en_US"
       iterator.text = "This is a sentence. This is another sentence, with a comma in it."
-      iterator.to_a.should == [0, 20, 65]
+      expect(iterator.to_a).to eq([0, 20, 65])
     end
 
     it "can navigate back and forward" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet."
 
-      iterator.first.should == 0
+      expect(iterator.first).to eq(0)
       iterator.next
-      iterator.current.should == 5
-      iterator.last.should == 27
+      expect(iterator.current).to eq(5)
+      expect(iterator.last).to eq(27)
     end
 
     it "fetches info about given offset" do
       iterator = BreakIterator.new :word, "en_US"
       iterator.text = "Lorem ipsum dolor sit amet."
 
-      iterator.following(3).should == 5
-      iterator.preceding(6).should == 5
+      expect(iterator.following(3)).to eq(5)
+      expect(iterator.preceding(6)).to eq(5)
 
-      iterator.should be_boundary(5)
-      iterator.should_not be_boundary(10)
+      expect(iterator).to be_boundary(5)
+      expect(iterator).to_not be_boundary(10)
     end
 
     it "returns an Enumerator if no block was given" do
       iterator = BreakIterator.new :word, "nb"
-      expected = ICU.ruby19? ? Enumerator : Enumerable::Enumerator
 
-      iterator.each.should be_kind_of(expected)
+      expect(iterator.each).to be_kind_of(Enumerator)
     end
 
   end # BreakIterator

data/spec/chardet_spec.rb

@@ -1,29 +1,27 @@
 # encoding: UTF-8
 
-require 'spec_helper'
-
 describe ICU::CharDet::Detector do
 
   let(:detector) { ICU::CharDet::Detector.new }
 
   it "should recognize UTF-8" do
     m = detector.detect("æåø")
-    m.name.should == "UTF-8"
-    m.language.should be_kind_of(String)
+    expect(m.name).to eq("UTF-8")
+    expect(m.language).to be_a(String)
   end
 
   it "has a list of detectable charsets" do
     cs = detector.detectable_charsets
-    cs.should be_kind_of(Array)
-    cs.should_not be_empty
+    expect(cs).to be_an(Array)
+    expect(cs).to_not be_empty
 
-    cs.first.should be_kind_of(String)
+    expect(cs.first).to be_a(String)
   end
 
   it "should disable / enable the input filter" do
-    detector.input_filter_enabled?.should be_false
+    expect(detector.input_filter_enabled?).to be_falsey
     detector.input_filter_enabled = true
-    detector.input_filter_enabled?.should be_true
+    expect(detector.input_filter_enabled?).to be_truthy
   end
 
   it "should should set declared encoding" do
@@ -31,14 +29,14 @@ describe ICU::CharDet::Detector do
   end
 
   it "should detect several matching encodings" do
-    detector.detect_all("foo bar").should be_instance_of(Array)
+    expect(detector.detect_all("foo bar")).to be_an(Array)
   end
 
   it "should support null bytes" do
     # Create a utf-16 string and then force it to binary (ascii) to mimic data from net/http
     string = "foo".encode("UTF-16").force_encoding("binary")
     m = detector.detect(string)
-    m.name.should == "UTF-16BE"
-    m.language.should be_kind_of(String)
+    expect(m.name).to eq("UTF-16BE")
+    expect(m.language).to be_a(String)
   end
 end

data/spec/collation_spec.rb

@@ -1,12 +1,10 @@
 # encoding: UTF-8
 
-require 'spec_helper'
-
 module ICU
   module Collation
     describe "Collation" do
       it "should collate an array of strings" do
-        Collation.collate("nb", %w[æ å ø]).should == %w[æ ø å]
+        expect(Collation.collate("nb", %w[æ å ø])).to eq(%w[æ ø å])
       end
     end
 
@@ -14,51 +12,50 @@ module ICU
       let(:collator) { Collator.new("nb") }
 
       it "should collate an array of strings" do
-        collator.collate(%w[å ø æ]).should == %w[æ ø å]
+        expect(collator.collate(%w[å ø æ])).to eq(%w[æ ø å])
       end
 
       it "raises an error if argument does not respond to :sort" do
-        lambda { collator.collate(1) }.should raise_error(ArgumentError)
+        expect { collator.collate(1) }.to raise_error(ArgumentError)
       end
 
       it "should return available locales" do
         locales = ICU::Collation.available_locales
-        locales.should be_kind_of(Array)
-        locales.should_not be_empty
-        locales.should include("nb")
+        expect(locales).to be_an(Array)
+        expect(locales).to_not be_empty
+        expect(locales).to include("nb")
       end
 
       it "should return the locale of the collator" do
-        l = collator.locale
-        l.should == "nb"
+        expect(collator.locale).to eq('nb')
       end
 
       it "should compare two strings" do
-        collator.compare("blåbærsyltetøy", "blah").should == 1
-        collator.compare("blah", "blah").should == 0
-        collator.compare("ba", "bl").should == -1
+        expect(collator.compare("blåbærsyltetøy", "blah")).to eq(1)
+        expect(collator.compare("blah", "blah")).to eq(0)
+        expect(collator.compare("ba", "bl")).to eq(-1)
       end
 
       it "should know if a string is greater than another" do
-        collator.should be_greater("z", "a")
-        collator.should_not be_greater("a", "z")
+        expect(collator).to be_greater("z", "a")
+        expect(collator).to_not be_greater("a", "z")
       end
 
       it "should know if a string is greater or equal to another" do
-        collator.should be_greater_or_equal("z", "a")
-        collator.should be_greater_or_equal("z", "z")
-        collator.should_not be_greater_or_equal("a", "z")
+        expect(collator).to be_greater_or_equal("z", "a")
+        expect(collator).to be_greater_or_equal("z", "z")
+        expect(collator).to_not be_greater_or_equal("a", "z")
       end
 
       it "should know if a string is equal to another" do
-        collator.should be_equal("a", "a")
-        collator.should_not be_equal("a", "b")
+        expect(collator).to be_equal("a", "a")
+        expect(collator).to_not be_equal("a", "b")
       end
 
       it "should return rules" do
-        collator.rules.should_not be_empty
+        expect(collator.rules).to_not be_empty
         # ö sorts before Ö
-        collator.rules.include?('ö<<<Ö').should be_true
+        expect(collator.rules).to include('ö<<<Ö')
       end
 
     end