ffi-icu 0.0.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Jari Bakken
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,71 @@
+= ffi-icu
+
+Simple FFI wrappers for things I need from ICU.
+
+= Dependencies
+
+ICU - you might need to hack the ffi_lib call to make it work. Please send a patch if you do!
+
+= Features
+
+== Character Encoding Detection
+
+=== Examples:
+
+  match = ICU::CharDet.detect(str)
+  match.name       # => "UTF-8"
+  match.confidence # => 80
+
+or
+
+  detector = ICU::CharDet::Detector.new
+  detector.detect(str)
+  detector.close
+
+=== Why not just use rchardet?
+
+* this is faster
+* rchardet does not work well on 1.9
+* none of the rchardet forks claiming to work on 1.9 actually does
+
+== Locale Sensitive Collation
+
+=== Examples:
+
+  ICU::Collation.collate("nb", %w[å æ ø]) == %w[æ ø å] #=> true
+
+or
+
+  collator = ICU::Collation::Collator.new("nb")
+  collator.compare("a", "b")  #=> -1
+  collator.greater?("z", "a") #=> true
+  collator.collate(%w[å æ ø]) #=> ["æ", "ø", "å"]
+
+= Tested on:
+
+Platforms:
+
+* OS X 10.6
+* Debian Linux
+* Arch Linux
+
+Rubies:
+
+* MRI 1.9.1
+* MRI 1.8.7
+
+YMMV.
+
+== Note on Patches/Pull Requests
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Jari Bakken. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name        = "ffi-icu"
+    gem.summary     = %Q{Simple FFI wrappers for things I need from ICU.}
+    gem.description = %Q{Provides charset detection, locale sensitive collation and more.}
+    gem.email       = "jari.bakken@gmail.com"
+    gem.homepage    = "http://github.com/jarib/ffi-icu"
+    gem.authors     = ["Jari Bakken"]
+
+    gem.add_dependency "ffi", ">= 0.6.3"
+    gem.add_development_dependency "rspec", ">= 1.3.0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/benchmark/detect.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+require "benchmark"
+
+$LOAD_PATH.unshift "lib"
+require "ffi-icu"
+require "rchardet"
+
+TESTS = 1000
+
+Benchmark.bmbm do |results|
+  results.report("rchardet:") { TESTS.times { CharDet.detect("æåø") } }
+  results.report("ffi-icu:") { TESTS.times { ICU::CharDet.detect("æåø") } }
+end

data/benchmark/shared.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+require "benchmark"
+
+$LOAD_PATH.unshift "lib"
+require "ffi-icu"
+require "rchardet"
+
+TESTS = 1000
+
+$rchardet = CharDet::UniversalDetector.new
+$icu = ICU::CharDet::Detector.new
+
+Benchmark.bmbm do |results|
+  results.report("rchardet instance:") { TESTS.times { $rchardet.reset; $rchardet.feed("æåø"); $rchardet.result } }
+  results.report("ffi-icu instance:") { TESTS.times { $icu.detect("æåø")  } }
+end

data/lib/ffi-icu.rb

@@ -0,0 +1,23 @@
+require "rbconfig"
+require "ffi"
+
+module ICU
+  def self.platform
+    os = RbConfig::CONFIG["host_os"]
+
+    case os
+    when /darwin/
+      :osx
+    when /linux/
+      :linux
+    else
+      os
+    end
+  end
+end
+
+require "ffi-icu/lib"
+require "ffi-icu/uchar"
+require "ffi-icu/chardet"
+require "ffi-icu/collation"
+

data/lib/ffi-icu/chardet.rb

@@ -0,0 +1,90 @@
+module ICU
+  module CharDet
+
+    def self.detect(string)
+      detector = Detector.new
+      res = detector.detect string
+      detector.close
+
+      res
+    end
+
+    class Detector
+      Match = Struct.new(:name, :confidence, :language)
+
+      def initialize
+        @detector = Lib.check_error { |ptr| Lib.ucsdet_open(ptr) }
+      end
+
+      def input_filter_enabled?
+        Lib.ucsdet_isInputFilterEnabled @detector
+      end
+
+      def input_filter_enabled=(bool)
+        Lib.ucsdet_enableInputFilter(@detector, !!bool)
+      end
+
+      def declared_encoding=(str)
+        Lib.check_error do |ptr|
+          Lib.ucsdet_setDeclaredEncoding(@detector, str, str.bytesize, ptr)
+        end
+      end
+
+      def close
+        Lib.ucsdet_close @detector
+      end
+
+      def detect(str)
+        set_text(str)
+
+        match_ptr = Lib.check_error { |ptr| Lib.ucsdet_detect(@detector, ptr) }
+        match_ptr_to_ruby(match_ptr) unless match_ptr.null?
+      end
+
+      def detect_all(str)
+        set_text(str)
+
+        matches_found_ptr = FFI::MemoryPointer.new :int32
+        array_ptr = Lib.check_error do |status|
+          Lib.ucsdet_detectAll(@detector, matches_found_ptr, status)
+        end
+
+        length = matches_found_ptr.read_int
+        array_ptr.read_array_of_pointer(length).map do |match|
+          match_ptr_to_ruby(match)
+        end
+      end
+
+      def detectable_charsets
+        enum_ptr = Lib.check_error do |ptr|
+          Lib.ucsdet_getAllDetectableCharsets(@detector, ptr)
+        end
+
+        result = Lib.enum_ptr_to_array(enum_ptr)
+        Lib.uenum_close(enum_ptr)
+
+        result
+      end
+
+      private
+
+      def match_ptr_to_ruby(match_ptr)
+        result = Match.new
+
+        result.name       = Lib.check_error { |ptr| Lib.ucsdet_getName(match_ptr, ptr) }
+        result.confidence = Lib.check_error { |ptr| Lib.ucsdet_getConfidence(match_ptr, ptr) }
+        result.language   = Lib.check_error { |ptr| Lib.ucsdet_getLanguage(match_ptr, ptr) }
+
+        result
+      end
+
+      def set_text(text)
+        Lib.check_error do |status|
+          Lib.ucsdet_setText(@detector, text, text.bytesize, status)
+        end
+      end
+
+    end # Detector
+  end # CharDet
+end # ICU
+

data/lib/ffi-icu/collation.rb

@@ -0,0 +1,78 @@
+module ICU
+  module Collation
+
+    def self.collate(locale, arr)
+      collator = Collator.new(locale)
+      res = collator.collate(arr)
+      collator.close
+
+      res
+    end
+
+    def self.keywords
+      enum_ptr = Lib.check_error { |error| Lib.ucol_getKeywords(error) }
+      keywords = Lib.enum_ptr_to_array(enum_ptr)
+      Lib.uenum_close enum_ptr
+
+      hash = {}
+      keywords.each do |keyword|
+        enum_ptr = Lib.check_error { |error| Lib.ucol_getKeywordValues(keyword, error) }
+        hash[keyword] = Lib.enum_ptr_to_array(enum_ptr)
+        Lib.uenum_close(enum_ptr)
+      end
+
+      hash
+    end
+
+    def self.available_locales
+      (0...Lib.ucol_countAvailable).map do |idx|
+        Lib.ucol_getAvailable idx
+      end
+    end
+
+    class Collator
+      ULOC_VALID_LOCALE = 1
+
+      def initialize(locale)
+        @c = Lib.check_error { |error| Lib.ucol_open(locale, error) }
+      end
+
+      def locale
+        Lib.check_error { |error| Lib.ucol_getLocale(@c, ULOC_VALID_LOCALE, error) }
+      end
+
+      def compare(a, b)
+        Lib.ucol_strcoll(
+          @c,
+          UCharPointer.from_string(a), a.length,
+          UCharPointer.from_string(b), b.length
+        )
+      end
+
+      def greater?(a, b)
+        Lib.ucol_greater(@c, UCharPointer.from_string(a), a.length,
+                             UCharPointer.from_string(b), b.length)
+      end
+
+      def greater_or_equal?(a, b)
+        Lib.ucol_greaterOrEqual(@c, UCharPointer.from_string(a), a.length,
+                                    UCharPointer.from_string(b), b.length)
+      end
+
+      # can't override Object#equal? - suggestions welcome
+      def same?(a, b)
+        Lib.ucol_equal(@c, UCharPointer.from_string(a), a.length,
+                           UCharPointer.from_string(b), b.length)
+      end
+
+      def collate(array)
+        array.sort { |a,b| compare a, b }
+      end
+
+      def close
+        Lib.ucol_close(@c)
+      end
+    end # Collator
+
+  end # Collate
+end # ICU

data/lib/ffi-icu/lib.rb

@@ -0,0 +1,112 @@
+module ICU
+  class Error < StandardError
+  end
+
+  module Lib
+    extend FFI::Library
+
+    VERSIONS = {
+      "42" => "_4_2",
+      "44" => "_44"
+    }
+
+    # FIXME: this is incredibly ugly, figure out some better way
+    def self.find_icu
+      suffix = ''
+
+      case ICU.platform
+      when :osx
+        ffi_lib "icucore"
+      when :linux
+        versions = VERSIONS.keys
+        libs = ffi_lib versions.map { |v| "libicui18n.so.#{v}"},
+                       versions.map { |v| "libicutu.so.#{v}"}
+
+        VERSIONS.find do |so_version, func_version|
+          if libs.first.name =~ /#{so_version}$/
+            suffix = func_version
+          end
+        end
+      else
+        raise "no idea how to load ICU on #{ICU.platform}, patches appreciated!"
+      end
+
+      suffix
+    end
+
+    def self.check_error
+      ptr = FFI::MemoryPointer.new(:int)
+      ret = yield(ptr)
+      error_code = ptr.read_int
+
+      if error_code > 0
+        raise Error, "#{Lib.u_errorName error_code}"
+      elsif error_code < 0
+        warn "ffi-icu: #{Lib.u_errorName error_code}"
+      end
+
+      ret
+    end
+
+    def self.enum_ptr_to_array(enum_ptr)
+      length = Lib.check_error do |status|
+        Lib.uenum_count(enum_ptr, status)
+      end
+
+      (0...length).map do |idx|
+        Lib.check_error { |st| Lib.uenum_next(enum_ptr, nil, st) }
+      end
+    end
+
+    def self.not_available(func_name)
+      self.class.send :define_method, func_name do |*args|
+        raise Error, "#{func_name} not available on platform #{ICU.platform.inspect}"
+      end
+    end
+
+
+    suffix = find_icu()
+
+    attach_function :u_errorName, "u_errorName#{suffix}", [:int], :string
+    attach_function :uenum_count, "uenum_count#{suffix}", [:pointer, :pointer], :int
+    attach_function :uenum_close, "uenum_close#{suffix}",  [:pointer], :void
+    attach_function :uenum_next, "uenum_next#{suffix}",  [:pointer, :pointer, :pointer], :string
+
+
+    # CharDet
+    #
+    # http://icu-project.org/apiref/icu4c/ucsdet_8h.html
+    #
+
+    attach_function :ucsdet_open, "ucsdet_open#{suffix}",  [:pointer], :pointer
+    attach_function :ucsdet_close, "ucsdet_close#{suffix}",  [:pointer], :void
+    attach_function :ucsdet_setText, "ucsdet_setText#{suffix}",  [:pointer, :string, :int32, :pointer], :void
+    attach_function :ucsdet_setDeclaredEncoding, "ucsdet_setDeclaredEncoding#{suffix}",  [:pointer, :string, :int32, :pointer], :void
+    attach_function :ucsdet_detect, "ucsdet_detect#{suffix}",  [:pointer, :pointer], :pointer
+    attach_function :ucsdet_detectAll, "ucsdet_detectAll#{suffix}",  [:pointer, :pointer, :pointer], :pointer
+    attach_function :ucsdet_getName, "ucsdet_getName#{suffix}",  [:pointer, :pointer], :string
+    attach_function :ucsdet_getConfidence, "ucsdet_getConfidence#{suffix}",  [:pointer, :pointer], :int32
+    attach_function :ucsdet_getLanguage, "ucsdet_getLanguage#{suffix}",  [:pointer, :pointer], :string
+    attach_function :ucsdet_getAllDetectableCharsets, "ucsdet_getAllDetectableCharsets#{suffix}",  [:pointer, :pointer], :pointer
+    attach_function :ucsdet_isInputFilterEnabled, "ucsdet_isInputFilterEnabled#{suffix}",  [:pointer], :bool
+    attach_function :ucsdet_enableInputFilter, "ucsdet_enableInputFilter#{suffix}",  [:pointer, :bool], :bool
+
+    # Collation
+    #
+    # http://icu-project.org/apiref/icu4c/ucol_8h.html
+    #
+
+    attach_function :ucol_open, "ucol_open#{suffix}",  [:string, :pointer], :pointer
+    attach_function :ucol_close, "ucol_close#{suffix}",  [:pointer], :void
+    attach_function :ucol_strcoll, "ucol_strcoll#{suffix}",  [:pointer, :pointer, :int32, :pointer, :int32], :int
+    attach_function :ucol_getKeywords, "ucol_getKeywords#{suffix}",  [:pointer], :pointer
+    attach_function :ucol_getKeywordValues, "ucol_getKeywordValues#{suffix}",  [:string, :pointer], :pointer
+    attach_function :ucol_getAvailable, "ucol_getAvailable#{suffix}", [:int32], :string
+    attach_function :ucol_countAvailable, "ucol_countAvailable#{suffix}", [], :int32
+    attach_function :ucol_getLocale, "ucol_getLocale#{suffix}", [:pointer, :int, :pointer], :string
+    attach_function :ucol_greater, "ucol_greater#{suffix}", [:pointer, :pointer, :int32, :pointer, :int32], :bool
+    attach_function :ucol_greaterOrEqual, "ucol_greaterOrEqual#{suffix}", [:pointer, :pointer, :int32, :pointer, :int32], :bool
+    attach_function :ucol_equal, "ucol_equal#{suffix}", [:pointer, :pointer, :int32, :pointer, :int32], :bool
+
+  end # Lib
+end # ICU

data/lib/ffi-icu/uchar.rb

@@ -0,0 +1,11 @@
+module ICU
+
+  class UCharPointer < FFI::MemoryPointer
+    def self.from_string(str)
+      # not sure how this will work with other encodings
+      str = str.encode("UTF-8") if str.respond_to? :encode
+      super str.unpack("U*").pack("L*")
+    end
+  end
+
+end

data/spec/chardet_spec.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+describe ICU::CharDet::Detector do
+
+  before { @d = ICU::CharDet::Detector.new }
+  after { @d.close }
+
+  it "should recognize UTF-8" do
+    m = @d.detect("æåø")
+    m.name.should == "UTF-8"
+    m.language.should be_kind_of(String)
+  end
+
+  it "has a list of detectable charsets" do
+    cs = @d.detectable_charsets
+    cs.should be_kind_of(Array)
+    cs.should_not be_empty
+
+    cs.first.should be_kind_of(String)
+  end
+
+  it "should disable / enable the input filter" do
+    @d.input_filter_enabled?.should be_false
+    @d.input_filter_enabled = true
+    @d.input_filter_enabled?.should be_true
+  end
+
+  it "should should set declared encoding" do
+    @d.declared_encoding = "UTF-8"
+  end
+
+  it "should detect several matching encodings" do
+    r = @d.detect_all("foo bar")
+    r.should be_instance_of(Array)
+  end
+
+end

data/spec/collation_spec.rb

@@ -0,0 +1,51 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+module ICU
+  module Collation
+    describe Collator do
+
+      before { @c = Collator.new("no") }
+      after { @c.close }
+
+      it "should collate an array of strings" do
+        @c.collate(%w[å ø æ]).should == %w[æ ø å]
+      end
+
+      it "should return available locales" do
+        locales = ICU::Collation.available_locales
+        locales.should be_kind_of(Array)
+        locales.should_not be_empty
+      end
+
+      it "should return the locale of the collator" do
+        l = @c.locale
+        l.should be_kind_of(String)
+        l.should == "nb"
+      end
+
+      it "should compare two strings" do
+        @c.compare("blåbærsyltetøy", "blah").should == 1
+        @c.compare("blah", "blah").should == 0
+        @c.compare("baah", "blah").should == -1
+      end
+
+      it "should know if a string is greater than another" do
+        @c.should be_greater("z", "a")
+        @c.should_not be_greater("a", "z")
+      end
+
+      it "should know if a string is greater or equal to another" do
+        @c.should be_greater_or_equal("z", "a")
+        @c.should be_greater_or_equal("z", "z")
+        @c.should_not be_greater_or_equal("a", "z")
+      end
+
+      it "should know if a string is equal to another" do
+        @c.should be_same("a", "a")
+      end
+
+    end
+  end # Collate
+end # ICU

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require "rubygems"
+require 'ffi-icu'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+
+end
+

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification 
+name: ffi-icu
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jari Bakken
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-11 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: ffi
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 6
+        - 3
+        version: 0.6.3
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :development
+  version_requirements: *id002
+description: Provides charset detection, locale sensitive collation and more.
+email: jari.bakken@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- benchmark/detect.rb
+- benchmark/shared.rb
+- lib/ffi-icu.rb
+- lib/ffi-icu/chardet.rb
+- lib/ffi-icu/collation.rb
+- lib/ffi-icu/lib.rb
+- lib/ffi-icu/uchar.rb
+- spec/chardet_spec.rb
+- spec/collation_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/jarib/ffi-icu
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Simple FFI wrappers for things I need from ICU.
+test_files: 
+- spec/chardet_spec.rb
+- spec/spec_helper.rb
+- spec/collation_spec.rb