ffi-icu 0.0.2 → 0.0.3

data/.gitignore

@@ -20,3 +20,4 @@ pkg
 
 ## PROJECT::SPECIFIC
 *.rbc
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Jari Bakken
+Copyright (c) 2010-2011 Jari Bakken
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -82,4 +82,4 @@ Rubies:
 
 == Copyright
 
-Copyright (c) 2010 Jari Bakken. See LICENSE for details.
+Copyright (c) 2010-2011 Jari Bakken. See LICENSE for details.

data/Rakefile

@@ -1,39 +1,20 @@
 require 'rubygems'
 require 'rake'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name        = "ffi-icu"
-    gem.summary     = %Q{Simple FFI wrappers for things I need from ICU.}
-    gem.description = %Q{Provides charset detection, locale sensitive collation and more.}
-    gem.email       = "jari.bakken@gmail.com"
-    gem.homepage    = "http://github.com/jarib/ffi-icu"
-    gem.authors     = ["Jari Bakken"]
-
-    gem.add_dependency "ffi", ">= 0.6.3"
-    gem.add_development_dependency "rspec", ">= 1.3.0"
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
-end
-
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
 end
 
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
+RSpec::Core::RakeTask.new(:rcov) do |spec|
   spec.pattern = 'spec/**/*_spec.rb'
   spec.rcov = true
 end
 
-task :spec => :check_dependencies
+task :spec
 
 task :default => :spec
 

data/ffi-icu.gemspec

@@ -0,0 +1,33 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+require File.expand_path("../lib/ffi-icu/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name                      = %q{ffi-icu}
+  s.version                   = ICU::VERSION
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors                   = ["Jari Bakken"]
+  s.date                      = %q{2010-08-23}
+  s.description               = %q{Provides charset detection, locale sensitive collation and more. Depends on libicu.}
+  s.email                     = %q{jari.bakken@gmail.com}
+  s.extra_rdoc_files          = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files                     = `git ls-files`.split("\n")
+  s.test_files                = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables               = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths             = ["lib"]
+  
+  s.homepage                  = %q{http://github.com/jarib/ffi-icu}
+  s.rdoc_options              = ["--charset=UTF-8"]
+  s.summary                   = %q{Simple FFI wrappers for things I need from ICU.}
+
+  s.add_runtime_dependency(%q<ffi>, ["~> 1.0.9"])
+  s.add_development_dependency(%q<rspec>, ["~> 2.5.0"])
+end
+

data/lib/ffi-icu.rb

@@ -20,12 +20,14 @@ module ICU
   end
 end
 
+require "ffi-icu/core_ext/string"
 require "ffi-icu/lib"
 require "ffi-icu/uchar"
 require "ffi-icu/chardet"
 require "ffi-icu/collation"
 require "ffi-icu/transliteration"
 require "ffi-icu/normalization"
+require "ffi-icu/break_iterator"
 
 unless ICU.ruby19?
   require 'jcode'

data/lib/ffi-icu/break_iterator.rb

@@ -0,0 +1,67 @@
+module ICU
+  class BreakIterator
+    include Enumerable
+
+    UBRK_DONE = -1
+
+    def self.available_locales
+      (0...Lib.ubrk_countAvailable).map do |idx|
+        Lib.ubrk_getAvailable idx
+      end
+    end
+
+    def initialize(type, locale)
+      ptr = Lib.check_error { |err| Lib.ubrk_open(type, locale, nil, 0, err) }
+
+      @iterator = FFI::AutoPointer.new(ptr, Lib.method(:ubrk_close))
+    end
+
+    def text=(str)
+      Lib.check_error { |err|
+        Lib.ubrk_setText @iterator, UCharPointer.from_string(str), str.length, err
+      }
+    end
+
+    def each(&blk)
+      int = first
+
+      while int != UBRK_DONE
+        yield int
+        int = self.next
+      end
+    end
+
+    def next
+      Lib.ubrk_next @iterator
+    end
+
+    def previous
+      Lib.ubrk_next @iterator
+    end
+
+    def first
+      Lib.ubrk_first @iterator
+    end
+
+    def last
+      Lib.ubrk_last @iterator
+    end
+
+    def preceding
+      Lib.ubrk_preceding @iterator
+    end
+
+    def following
+      Lib.ubrk_following @iterator
+    end
+
+    def current
+      Lib.ubrk_current @iterator
+    end
+
+    def boundary?(index)
+      Lib.ubrk_isBoundary(@iterator, Integer(index)) != 0
+    end
+
+  end # BreakIterator
+end # ICU

data/lib/ffi-icu/core_ext/string.rb

@@ -0,0 +1,5 @@
+class String
+  unless method_defined?(:bytesize)
+    alias_method :bytesize, :length
+  end
+end

data/lib/ffi-icu/lib.rb

@@ -20,7 +20,7 @@ module ICU
       # let the user tell us where the lib is
       if ENV['FFI_ICU_LIB']
         libs = ENV['FFI_ICU_LIB'].split(",")
-        ffi_lib *libs
+        ffi_lib(*libs)
 
         if ENV['FFI_ICU_VERSION_SUFFIX']
           return ENV['FFI_ICU_VERSION_SUFFIX']
@@ -145,7 +145,9 @@ module ICU
              :pre_context,  :pointer,
              :post_context, :pointer
 
-
+      def to_s
+        "#<%s:%x line: %d offset: %d" % [self.class, hash*2, self[:line], self[:offset]]
+      end
     end
 
     class UTransPosition < FFI::Struct
@@ -179,5 +181,34 @@ module ICU
                               ]
 
     attach_function :unorm_normalize, "unorm_normalize#{suffix}", [:pointer, :int32_t, :normalization_mode, :int32_t, :pointer, :int32_t, :pointer], :int32_t
+
+    #
+    # Text Boundary Analysis
+    #
+
+    enum :iterator_type, [ :character, :word, :line, :sentence, :title]
+    enum :word_break, [ :none,         0,
+                        :none_limit,   100,
+                        :number,       100,
+                        :number_limit, 200,
+                        :letter,       200,
+                        :letter_limit, 300,
+                        :kana,         300,
+                        :kana_limit,   400,
+                        :ideo,         400,
+                        :ideo_limit,   400
+                      ]
+
+    attach_function :ubrk_countAvailable, "ubrk_countAvailable#{suffix}", [], :int32_t
+    attach_function :ubrk_getAvailable, "ubrk_getAvailable#{suffix}", [:int32_t], :string
+
+    attach_function :ubrk_open, "ubrk_open#{suffix}", [:iterator_type, :string, :pointer, :int32_t, :pointer], :pointer
+    attach_function :ubrk_close, "ubrk_close#{suffix}", [:pointer], :void
+    attach_function :ubrk_setText, "ubrk_setText#{suffix}", [:pointer, :pointer, :int32_t, :pointer], :void
+    attach_function :ubrk_current, "ubrk_current#{suffix}", [:pointer], :int32_t
+    attach_function :ubrk_next, "ubrk_next#{suffix}", [:pointer], :int32_t
+    attach_function :ubrk_previous, "ubrk_previous#{suffix}", [:pointer], :int32_t
+    attach_function :ubrk_first, "ubrk_first#{suffix}", [:pointer], :int32_t
+    attach_function :ubrk_last, "ubrk_last#{suffix}", [:pointer], :int32_t
   end # Lib
 end # ICU

data/lib/ffi-icu/normalization.rb

@@ -2,27 +2,28 @@ module ICU
   module Normalization
 
     def self.normalize(input, mode = :default)
-      input_length = ICU.ruby19? ? input.length : input.jlength
-      needed_length  = 0
-      result_length = 0
+      input_length  = input.unpack("U*").size
+      needed_length = out_length = options = 0
+      in_ptr        = UCharPointer.from_string(input)
+      out_ptr       = UCharPointer.new(out_length)
 
       retried = false
-      ptr     = nil
 
       begin
         Lib.check_error do |error|
-          needed_length = Lib.unorm_normalize(UCharPointer.from_string(input), input_length, mode, 0, ptr, result_length, error)
+          needed_length = Lib.unorm_normalize(in_ptr, input_length, mode, options, out_ptr, out_length, error)
         end
       rescue BufferOverflowError
-        raise if retried
-        ptr     = UCharPointer.from_string("\0" * needed_length)
-        result_length = needed_length + 1
+        raise BufferOverflowError, "needed: #{needed_length}" if retried
+
+        out_ptr       = out_ptr.resized_to needed_length
+        out_length    = needed_length + 1
 
         retried = true
         retry
       end
 
-      ptr.string if ptr
+      out_ptr.string
     end
 
   end # Normalization

data/lib/ffi-icu/transliteration.rb

@@ -2,8 +2,8 @@ module ICU
   module Transliteration
 
     class << self
-      def transliterate(translit_id, str)
-        t = Transliterator.new translit_id
+      def transliterate(translit_id, str, rules = nil)
+        t = Transliterator.new translit_id, rules
         res = t.transliterate str
         t.close
 
@@ -25,11 +25,22 @@ module ICU
 
     class Transliterator
 
-      def initialize(id, direction = :forward)
-        @parse_error = Lib::UParseError.new
-        Lib.check_error do |status|
-          # couldn't get utrans_openU to work properly, so using deprecated utrans_open for now
-          @tr = Lib.utrans_open(id, direction, nil, 0, @parse_error, status)
+      def initialize(id, rules = nil, direction = :forward)
+        if rules
+          rules_length = rules.length + 1
+          rules = UCharPointer.from_string(rules)
+        else
+          rules_length = 0
+        end
+
+        parse_error = Lib::UParseError.new
+        begin
+          Lib.check_error do |status|
+            # couldn't get utrans_openU to work properly, so using deprecated utrans_open for now
+            @tr = Lib.utrans_open(id, direction, rules, rules_length, @parse_error, status)
+          end
+        rescue ICU::Error => ex
+          raise ex, "#{ex.message} (#{parse_error})"
         end
       end
 

data/lib/ffi-icu/uchar.rb

@@ -8,15 +8,19 @@ module ICU
       str   = str.encode("UTF-8") if str.respond_to? :encode
       bytes = str.unpack("U*")
 
-      ptr = new UCHAR_TYPE, bytes.size
+      ptr = new bytes.size
       ptr.put_array_of_uint16 0, bytes
 
       ptr
     end
 
+    def initialize(size)
+      super UCHAR_TYPE, size
+    end
+
     def resized_to(new_size)
       raise "new_size must be larger than current size" if new_size < size
-      resized = self.class.new UCHAR_TYPE, new_size
+      resized = self.class.new new_size
       resized.put_bytes(0, get_bytes(0, size))
 
       resized

data/lib/ffi-icu/version.rb

@@ -0,0 +1,3 @@
+module ICU
+  VERSION = "0.0.3"
+end

data/spec/break_iterator_spec.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+require "spec_helper"
+
+module ICU
+  describe BreakIterator do
+
+    it "should return available locales" do
+      locales = ICU::BreakIterator.available_locales
+      locales.should be_kind_of(Array)
+      locales.should_not be_empty
+      locales.should include("en_US")
+    end
+
+    it "finds all word boundaries in an English string" do
+      iterator = BreakIterator.new :word, "en_US"
+      iterator.text = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."
+      iterator.to_a.should == [0, 5, 6, 11, 12, 17, 18, 21, 22, 26, 27, 28, 39, 40, 51, 52, 56, 57, 58, 61, 62, 64, 65, 72, 73, 79, 80, 90, 91, 93, 94, 100, 101, 103, 104, 110, 111, 116, 117, 123, 124]
+    end
+
+    it "finds all sentence boundaries in an English string" do
+      iterator = BreakIterator.new :sentence, "en_US"
+      iterator.text = "This is a sentence. This is another sentence, with a comma in it."
+      iterator.to_a.should == [0, 20, 65]
+    end
+
+  end # BreakIterator
+end # ICU

data/spec/collation_spec.rb

@@ -17,6 +17,7 @@ module ICU
         locales = ICU::Collation.available_locales
         locales.should be_kind_of(Array)
         locales.should_not be_empty
+        locales.should include("nb")
       end
 
       it "should return the locale of the collator" do

data/spec/spec_helper.rb

@@ -3,10 +3,9 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 
 require "rubygems"
 require 'ffi-icu'
-require 'spec'
-require 'spec/autorun'
+require 'rspec'
 
-Spec::Runner.configure do |config|
+RSpec.configure do |config|
 
 end
 

data/spec/transliteration_spec.rb

@@ -29,5 +29,10 @@ module ICU
       ids.should be_kind_of(Array)
       ids.should_not be_empty
     end
+
+#    it "should transliterate custom rules" do
+#      ICU::Transliteration.translit("Accents-Any", "âêîôû", "NFD; [:Nonspacing Mark:] Remove; NFC").should == "aeiou"
+    #    end
+
   end # Transliteration
 end # ICU

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: ffi-icu
 version: !ruby/object:Gem::Version 
-  hash: 27
-  prerelease: false
-  segments: 
-  - 0
-  - 0
-  - 2
-  version: 0.0.2
+  prerelease: 
+  version: 0.0.3
 platform: ruby
 authors: 
 - Jari Bakken
@@ -15,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-29 00:00:00 +02:00
-default_executable: 
+date: 2010-08-23 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: ffi
@@ -24,14 +18,9 @@ dependencies:
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 0
-        - 6
-        - 3
-        version: 0.6.3
+        version: 1.0.9
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -40,17 +29,12 @@ dependencies:
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 1
-        - 3
-        - 0
-        version: 1.3.0
+        version: 2.5.0
   type: :development
   version_requirements: *id002
-description: Provides charset detection, locale sensitive collation and more.
+description: Provides charset detection, locale sensitive collation and more. Depends on libicu.
 email: jari.bakken@gmail.com
 executables: []
 
@@ -62,19 +46,24 @@ extra_rdoc_files:
 files: 
 - .document
 - .gitignore
+- Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
-- VERSION
 - benchmark/detect.rb
 - benchmark/shared.rb
+- ffi-icu.gemspec
 - lib/ffi-icu.rb
+- lib/ffi-icu/break_iterator.rb
 - lib/ffi-icu/chardet.rb
 - lib/ffi-icu/collation.rb
+- lib/ffi-icu/core_ext/string.rb
 - lib/ffi-icu/lib.rb
 - lib/ffi-icu/normalization.rb
 - lib/ffi-icu/transliteration.rb
 - lib/ffi-icu/uchar.rb
+- lib/ffi-icu/version.rb
+- spec/break_iterator_spec.rb
 - spec/chardet_spec.rb
 - spec/collation_spec.rb
 - spec/normalization_spec.rb
@@ -82,7 +71,6 @@ files:
 - spec/spec_helper.rb
 - spec/transliteration_spec.rb
 - test.c
-has_rdoc: true
 homepage: http://github.com/jarib/ffi-icu
 licenses: []
 
@@ -96,29 +84,25 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.2
 signing_key: 
 specification_version: 3
 summary: Simple FFI wrappers for things I need from ICU.
 test_files: 
+- spec/break_iterator_spec.rb
 - spec/chardet_spec.rb
+- spec/collation_spec.rb
 - spec/normalization_spec.rb
-- spec/transliteration_spec.rb
+- spec/spec.opts
 - spec/spec_helper.rb
-- spec/collation_spec.rb
+- spec/transliteration_spec.rb

data/VERSION

@@ -1 +0,0 @@
-0.0.2