ffi-icu 0.0.6 → 0.0.7

data/README.md

@@ -0,0 +1,116 @@
+ffi-icu
+=======
+
+Simple FFI wrappers for things I need from ICU. For the full thing, check out [ICU4R](http://icu4r.rubyforge.org/) instead.
+
+Gem
+---
+
+[Rubygem](http://rubygems.org/gems/ffi-icu "ffi-icu")
+
+    gem install ffi-icu
+
+Dependencies
+------------
+
+ICU. 
+
+If you get messages that the library or functions are not found, you can
+set some environment varibles to tell ffi-icu where to find it, e.g.:
+  
+    $ export FFI_ICU_LIB="icui18n.so"
+    $ export FFI_ICU_VERSION_SUFFIX="_3_8" 
+    $ ruby -r ffi-icu program.rb
+
+Features
+========
+
+Character Encoding Detection
+----------------------------
+
+Examples:
+
+```ruby
+
+    match = ICU::CharDet.detect(str)
+    match.name       # => "UTF-8"
+    match.confidence # => 80
+```
+
+or
+
+```ruby
+    detector = ICU::CharDet::Detector.new
+    detector.detect(str) => #<struct ICU::CharDet::Detector::Match ...>
+```      
+
+Why not just use rchardet?
+
+* speed
+* 1.9 support
+
+Locale Sensitive Collation
+--------------------------
+
+Examples:
+
+```ruby
+    ICU::Collation.collate("nb", %w[å æ ø]) == %w[æ ø å] #=> true
+```
+
+or
+
+```ruby
+    collator = ICU::Collation::Collator.new("nb")
+    collator.compare("a", "b")  #=> -1
+    collator.greater?("z", "a") #=> true
+    collator.collate(%w[å æ ø]) #=> ["æ", "ø", "å"]
+```
+  
+Text Boundary Analysis
+----------------------
+
+Examples:
+
+```ruby
+    iterator = ICU::BreakIterator.new(:word, "en_US")
+    iterator.text = "This is a sentence."
+    iterator.to_a  #=> [0, 4, 5, 7, 8, 9, 10, 18, 19]
+```
+
+Tested on:
+==========
+
+Platforms:
+
+* OS X 10.6
+* Arch Linux
+
+Rubies:
+
+* MRI 1.9.1
+* MRI 1.8.7
+
+TODO:
+=====
+
+* Useful ICU stuff:
+  - number formatting (decimal points, thousand separators, currency)
+  - date formatting
+* Windows?!
+
+Note on Patches/Pull Requests
+=============================
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+Copyright
+=========
+
+Copyright (c) 2010-2011 Jari Bakken. See LICENSE for details.

data/Rakefile

@@ -21,7 +21,7 @@ task :default => :spec
 begin
   require 'yard'
   YARD::Rake::YardocTask.new
-rescue LoadError
+rescue
   task :yardoc do
     abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
   end

data/ffi-icu.gemspec

@@ -14,18 +14,15 @@ Gem::Specification.new do |s|
   s.date                      = %q{2010-08-23}
   s.description               = %q{Provides charset detection, locale sensitive collation and more. Depends on libicu.}
   s.email                     = %q{jari.bakken@gmail.com}
-  s.extra_rdoc_files          = [
-    "LICENSE",
-     "README.rdoc"
-  ]
+  s.extra_rdoc_files          = ["LICENSE", "README.md"]
   s.files                     = `git ls-files`.split("\n")
   s.test_files                = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables               = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths             = ["lib"]
-  
+
   s.homepage                  = %q{http://github.com/jarib/ffi-icu}
   s.rdoc_options              = ["--charset=UTF-8"]
-  s.summary                   = %q{Simple FFI wrappers for things I need from ICU.}
+  s.summary                   = %q{Simple Ruby FFI wrappers for things I need from ICU.}
 
   s.add_runtime_dependency(%q<ffi>, ["~> 1.0.9"])
   s.add_development_dependency(%q<rspec>, ["~> 2.5.0"])

data/lib/ffi-icu/break_iterator.rb

@@ -2,7 +2,9 @@ module ICU
   class BreakIterator
     include Enumerable
 
-    UBRK_DONE = -1
+    attr_reader :text
+
+    DONE = -1
 
     def self.available_locales
       (0...Lib.ubrk_countAvailable).map do |idx|
@@ -12,11 +14,12 @@ module ICU
 
     def initialize(type, locale)
       ptr = Lib.check_error { |err| Lib.ubrk_open(type, locale, nil, 0, err) }
-
       @iterator = FFI::AutoPointer.new(ptr, Lib.method(:ubrk_close))
     end
 
     def text=(str)
+      @text = str
+
       Lib.check_error { |err|
         Lib.ubrk_setText @iterator, UCharPointer.from_string(str), str.jlength, err
       }
@@ -27,10 +30,31 @@ module ICU
 
       int = first
 
-      while int != UBRK_DONE
+      while int != DONE
         yield int
         int = self.next
       end
+
+      self
+    end
+
+    def each_substring(&blk)
+      return to_enum(:each_substring) unless block_given?
+
+      # each_char needed for 1.8, where String#[] works on bytes, not characters
+      chars = text.each_char.to_a
+      low   = first
+
+      while (high = self.next) != DONE
+        yield chars[low...high].join
+        low = high
+      end
+
+      self
+    end
+
+    def substrings
+      each_substring.to_a
     end
 
     def next

data/lib/ffi-icu/lib.rb

@@ -77,7 +77,7 @@ module ICU
           raise Error, name
         end
       elsif error_code < 0
-        warn "ffi-icu: #{Lib.u_errorName error_code}"
+        $stderr.puts "ffi-icu: #{Lib.u_errorName error_code}" if $DEBUG || $VERBOSE
       end
 
       ret

data/lib/ffi-icu/version.rb

@@ -1,3 +1,3 @@
 module ICU
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/break_iterator_spec.rb

@@ -18,7 +18,21 @@ module ICU
       iterator.to_a.should == [0, 5, 6, 11, 12, 17, 18, 21, 22, 26, 27, 28, 39, 40, 51, 52, 56, 57, 58, 61, 62, 64, 65, 72, 73, 79, 80, 90, 91, 93, 94, 100, 101, 103, 104, 110, 111, 116, 117, 123, 124]
     end
 
-    it "finds all word boundaries in a Thai string" do
+    it "returns each substring" do
+      iterator = BreakIterator.new :word, "en_US"
+      iterator.text = "Lorem ipsum dolor sit amet."
+
+      iterator.substrings.should == ["Lorem", " ", "ipsum", " ", "dolor", " ", "sit", " ", "amet", "."]
+    end
+
+    it "returns the substrings of a non-ASCII string" do
+      iterator = BreakIterator.new :word, "th_TH"
+      iterator.text = "รู้อะไรไม่สู้รู้วิชา รู้รักษาตัวรอดเป็นยอดดี"
+
+      iterator.substrings.should ==  ["รู้", "อะไร", "ไม่สู้", "รู้", "วิชา", " ", "รู้", "รักษา", "ตัว", "รอด", "เป็น", "ยอดดี"]
+    end
+
+    it "finds all word boundaries in a non-ASCII string" do
       iterator = BreakIterator.new :word, "th_TH"
       iterator.text = "การทดลอง"
       iterator.to_a.should == [0, 3, 8]

metadata

@@ -2,7 +2,7 @@
 name: ffi-icu
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors: 
 - Jari Bakken
@@ -42,13 +42,13 @@ extensions: []
 
 extra_rdoc_files: 
 - LICENSE
-- README.rdoc
+- README.md
 files: 
 - .document
 - .gitignore
 - Gemfile
 - LICENSE
-- README.rdoc
+- README.md
 - Rakefile
 - benchmark/detect.rb
 - benchmark/shared.rb
@@ -97,7 +97,7 @@ rubyforge_project:
 rubygems_version: 1.8.2
 signing_key: 
 specification_version: 3
-summary: Simple FFI wrappers for things I need from ICU.
+summary: Simple Ruby FFI wrappers for things I need from ICU.
 test_files: 
 - spec/break_iterator_spec.rb
 - spec/chardet_spec.rb

data/README.rdoc

@@ -1,85 +0,0 @@
-= ffi-icu
-
-Simple FFI wrappers for things I need from ICU. For the full thing, check out ICU4R instead.
-
-= Gem
-
-* http://rubygems.org/gems/ffi-icu
-
-  gem install ffi-icu
-
-= Dependencies
-
-ICU. If you get messages that the library or functions are not found, you can
-set some environment varibles to tell ffi-icu where to find it, i.e.:
-
-  FFI_ICU_LIB="icui18n.so" FFI_ICU_VERSION_SUFFIX="_3_8" ruby -r ffi-icu
-
-= Features
-
-== Character Encoding Detection
-
-=== Examples:
-
-  match = ICU::CharDet.detect(str)
-  match.name       # => "UTF-8"
-  match.confidence # => 80
-
-or
-
-  detector = ICU::CharDet::Detector.new
-  detector.detect(str)
-  detector.close
-
-=== Why not just use rchardet?
-
-* this is faster
-* rchardet does not work well on 1.9
-* none of the rchardet forks claiming to work on 1.9 actually does
-
-== Locale Sensitive Collation
-
-=== Examples:
-
-  ICU::Collation.collate("nb", %w[å æ ø]) == %w[æ ø å] #=> true
-
-or
-
-  collator = ICU::Collation::Collator.new("nb")
-  collator.compare("a", "b")  #=> -1
-  collator.greater?("z", "a") #=> true
-  collator.collate(%w[å æ ø]) #=> ["æ", "ø", "å"]
-
-= Tested on:
-
-Platforms:
-
-* OS X 10.6
-* Debian Linux
-* Arch Linux
-
-Rubies:
-
-* MRI 1.9.1
-* MRI 1.8.7
-
-= TODO:
-
-* Useful ICU stuff:
-  - number formatting (decimal points, thousand separators, currency)
-  - date formatting
-* Windows?!
-
-== Note on Patches/Pull Requests
-
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
-
-== Copyright
-
-Copyright (c) 2010-2011 Jari Bakken. See LICENSE for details.