utf8 0.1.6 → 0.1.7

data/.travis.yml

@@ -0,0 +1,6 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - rbx
+  - ree

data/benchmark/active_support.rb

@@ -5,6 +5,7 @@ require 'benchmark'
 require 'rubygems'
 require 'active_support'
 
+$KCODE = 'UTF8'
 
 raw = File.read(File.expand_path('../test.txt', __FILE__))
 utf8 = raw.as_utf8
@@ -29,9 +30,15 @@ Benchmark.bmbm { |x|
   x.report("#[-start, len]") {
     times.times {utf8[-1024, 1024]}
   }
+  x.report("#clean") {
+    times.times {utf8.clean}
+  }
+  x.report("#valid?") {
+    times.times {utf8.valid?}
+  }
 }
 
-puts "\n\nActiveSupport::Multibyte::Chars"
+puts "\n\nActiveSupport::Multibyte"
 Benchmark.bmbm { |x|
   x.report("#length") {
     times.times {as_mb.length}
@@ -48,4 +55,35 @@ Benchmark.bmbm { |x|
   x.report("#[-start, len]") {
     times.times {as_mb[-1024, 1024]}
   }
+  x.report("ActiveSupport::Multibyte.clean") {
+    times.times {ActiveSupport::Multibyte.clean(raw)}
+  }
+  x.report("ActiveSupport::Multibyte.verify") {
+    times.times {ActiveSupport::Multibyte.verify(raw)}
+  }
+}
+
+require 'iconv'
+module ActiveSupport::Multibyte
+  class << self
+    OUTSIDE_ASCII = /[^\x00-\x7f]/n
+    ICONV_CLEANER = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+
+    def clean_with_iconv(string)
+      if string =~ OUTSIDE_ASCII
+        ICONV_CLEANER.iconv(string + ' ')[0..-2]
+      else
+        string
+      end
+    end
+
+    alias_method :clean_without_iconv, :clean
+    alias_method :clean, :clean_with_iconv
+  end
+end
+puts "\n\nActiveSupport::Multibyte (patched with Iconv)"
+Benchmark.bmbm { |x|
+  x.report("ActiveSupport::Multibyte.clean") {
+    times.times {ActiveSupport::Multibyte.clean(raw)}
+  }
 }

data/ext/utf8/extconf.rb

@@ -4,4 +4,4 @@ require 'rbconfig'
 $CFLAGS << ' -Wall -funroll-loops'
 $CFLAGS << ' -Wextra -O0 -ggdb3' if ENV['DEBUG']
 
-create_makefile("utf8")
+create_makefile("utf8/utf8")

data/ext/utf8/string_utf8.c

@@ -93,7 +93,7 @@ static VALUE rb_cString_UTF8_each_codepoint(int argc, VALUE *argv, VALUE self) {
 /*
  * call-seq: valid?(max_codepoint=nil)
  *
- * Iterates over the string, yielding one UTF-8 codepoint at a time
+ * Iterates over the string, returning true/false if it's valid UTF-8
  *
  * max_codepoint - an optional Fixnum used to declare this string invalid
  *                 if a codepoint higher than that value is found
@@ -337,44 +337,73 @@ static VALUE rb_cString_UTF8_slice(int argc, VALUE *argv, VALUE self) {
  * Returns: a new String
  */
 static VALUE rb_cString_UTF8_clean(VALUE self) {
-  unsigned char *str;
-  unsigned char *out;
-  unsigned char replace;
-  size_t len;
+  unsigned char *inBuf, *inBufCur;
+  unsigned char *outBuf, *outBufCur;
+  size_t len, i;
   int8_t curCharLen;
-  size_t i;
   VALUE rb_out;
 
-  str = (unsigned char *)RSTRING_PTR(self);
+  inBuf = (unsigned char *)RSTRING_PTR(self);
+  inBufCur = inBuf;
   len = RSTRING_LEN(self);
-  replace = REPLACEMENT_CHAR;
-  out = xmalloc(len);
+  outBuf = malloc(len);
+  outBufCur = outBuf;
 
   for(i=0; i<len; i+=curCharLen) {
-    curCharLen = utf8CharLen(str+i, len);
+    curCharLen = utf8CharLen(inBufCur, len);
     if (curCharLen < 0) {
-      *(out+i) = replace;
+      if (inBufCur-inBuf > 0) {
+        memcpy(outBufCur, inBuf, inBufCur-inBuf);
+        outBufCur += inBufCur-inBuf;
+      }
+      *outBufCur++ = REPLACEMENT_CHAR;
+      inBuf += (inBufCur-inBuf)+1;
       curCharLen = 1;
-    } else {
-      memcpy(out+i, str+i, curCharLen);
     }
+
+    inBufCur += curCharLen;
+  }
+
+  if (inBufCur-inBuf > 0) {
+    memcpy(outBufCur, inBuf, inBufCur-inBuf);
   }
 
-  rb_out = rb_str_new((const char*)out, len);
+  rb_out = rb_str_new((const char*)outBuf, len);
   AS_UTF8(rb_out);
 
-  xfree(out);
+  free(outBuf);
 
   return rb_out;
 }
 
+/*
+ * call-seq: clean
+ *
+ * Iterates over the string, returning true/false if it's within the low ASCII range
+ *
+ * Returns: a Boolean - true if the string is within the low ASCII range, false if not
+ */
+static VALUE rb_cString_UTF8_ascii_only(VALUE self) {
+  unsigned char *str = (unsigned char *)RSTRING_PTR(self);
+  size_t len = RSTRING_LEN(self), i=0;
+
+  for(; i<len; i+=1) {
+    if (str[i] > 0x7f) {
+      return Qfalse;
+    }
+  }
+
+  return Qtrue;
+}
+
 void init_String_UTF8() {
   VALUE rb_cString_UTF8 = rb_define_class_under(rb_cString, "UTF8", rb_cString);
 
-  rb_define_method(rb_cString_UTF8, "length",    rb_cString_UTF8_length, 0);
+  rb_define_method(rb_cString_UTF8, "length", rb_cString_UTF8_length, 0);
   rb_define_method(rb_cString_UTF8, "each_char", rb_cString_UTF8_each_char, -1);
-  rb_define_method(rb_cString_UTF8, "[]",        rb_cString_UTF8_slice, -1);
+  rb_define_method(rb_cString_UTF8, "[]", rb_cString_UTF8_slice, -1);
   rb_define_method(rb_cString_UTF8, "each_codepoint", rb_cString_UTF8_each_codepoint, -1);
   rb_define_method(rb_cString_UTF8, "valid?", rb_cString_UTF8_valid, -1);
   rb_define_method(rb_cString_UTF8, "clean", rb_cString_UTF8_clean, 0);
+  rb_define_method(rb_cString_UTF8, "ascii_only?", rb_cString_UTF8_ascii_only, 0);
 }

data/ext/utf8/utf8.c

@@ -7,7 +7,7 @@
  * Scans the current position of the buffer
  * returning the length of this UTF-8 character
  */
-inline int8_t utf8CharLen(unsigned char *in, size_t in_len) {
+int8_t utf8CharLen(unsigned char *in, size_t in_len) {
   if (in_len > 0) {
     unsigned char curChar, *start;
 

data/ext/utf8/utf8.h

@@ -1,7 +1,7 @@
 #ifndef UTF8_UTF8_H
 #define UTF8_UTF8_H
 
-inline int8_t utf8CharLen(unsigned char *in, size_t in_len);
+int8_t utf8CharLen(unsigned char *in, size_t in_len);
 int64_t utf8CharCount(unsigned char *in, size_t in_len);
 int32_t utf8CharToCodepoint(unsigned char *in, size_t in_len);
 

data/lib/utf8/version.rb

@@ -1,5 +1,5 @@
 class String
   class UTF8 < ::String
-    VERSION = "0.1.6"
+    VERSION = "0.1.7"
   end
 end

data/spec/spec_helper.rb

@@ -2,4 +2,9 @@ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'utf8'
 require 'utf8/string_scanner'
 
-require 'rspec' unless defined? Rspec
+require 'rspec' unless defined? RSpec
+
+RSpec.configure do |config|
+  config.expect_with :stdlib
+  config.alias_example_to :test
+end

data/spec/string_scanner_spec.rb

@@ -2,13 +2,13 @@
 require File.expand_path('../spec_helper', __FILE__)
 
 describe StringScanner::UTF8 do
-  before(:each) do
+  before :each do
     @char_array = ["怎", "麼", "也", "沒", "人", "寫", "了", "這", "個", "嗎"]
     @scanner = StringScanner.new(@char_array.join)
     @utf8_scanner = @scanner.as_utf8
   end
 
-  it "should blow up on invalid utf8 chars" do
+  test "should blow up on invalid utf8 chars" do
     # lets cut right into the middle of a sequence so we know it's bad
     str = @char_array.join
     str.force_encoding('binary') if str.respond_to?(:force_encoding)
@@ -16,34 +16,34 @@ describe StringScanner::UTF8 do
     str.force_encoding('utf-8') if str.respond_to?(:force_encoding)
     scanner = StringScanner.new(str).as_utf8
 
-    lambda {
+    assert_raise ArgumentError do
       scanner.getch
-    }.should raise_error(ArgumentError)
+    end
   end
 
-  it "should extend StringScanner, adding an as_utf8 method that returns a StringScanner::UTF8 instance" do
-    @scanner.should respond_to(:as_utf8)
-    @scanner.as_utf8.class.should eql(StringScanner::UTF8)
+  test "should extend StringScanner, adding an as_utf8 method that returns a StringScanner::UTF8 instance" do
+    assert @scanner.respond_to?(:as_utf8)
+    assert_equal StringScanner::UTF8, @scanner.as_utf8.class
   end
 
-  it "should allow access to a regular (non-utf8-aware) StringScanner based on it's string" do
+  test "should allow access to a regular (non-utf8-aware) StringScanner based on it's string" do
     raw = @utf8_scanner.as_raw
-    raw.class.should eql(StringScanner)
-    raw.string.should eql(@utf8_scanner.string)
+    assert_equal StringScanner, raw.class
+    assert_equal @utf8_scanner.string, raw.string
   end
 
-  it "#getch should be utf8-aware" do
+  test "#getch should be utf8-aware" do
     i=0
     while char = @utf8_scanner.getch
-      char.should eql(@char_array[i])
+      assert_equal @char_array[i], char
       i+=1
     end
   end
 
-  it "should be able to be reset" do
+  test "should be able to be reset" do
     i=0
     while char = @utf8_scanner.getch
-      char.should eql(@char_array[i])
+      assert_equal @char_array[i], char
       if i == 4
         break
       end
@@ -54,7 +54,7 @@ describe StringScanner::UTF8 do
 
     i=0
     while char = @utf8_scanner.getch
-      char.should eql(@char_array[i])
+      assert_equal @char_array[i], char
       i+=1
     end
   end

data/spec/string_spec.rb

@@ -2,7 +2,7 @@
 require File.expand_path('../spec_helper', __FILE__)
 
 describe String::UTF8 do
-  before(:each) do
+  before :each do
     @char_array = ["怎", "麼", "也", "沒", "人", "寫", "了", "這", "個", "嗎"]
     @str = @char_array.join
     @utf8 = @str.as_utf8
@@ -10,65 +10,69 @@ describe String::UTF8 do
     @codepoints = @char_array.map{|c| c.unpack('U').first}
   end
 
-  it "should blow up on invalid utf8 chars" do
+  test "should blow up on invalid utf8 chars" do
     # lets cut right into the middle of a sequence so we know it's bad
     @str.force_encoding('binary') if @str.respond_to?(:force_encoding)
     utf8 = @str[0..1]
     utf8.force_encoding('utf-8') if utf8.respond_to?(:force_encoding)
     utf8 = utf8.as_utf8
 
-    lambda {
+    assert_raise ArgumentError do
       utf8.length
-    }.should raise_error(ArgumentError)
+    end
 
-    lambda {
+    assert_raise ArgumentError do
       utf8[0, 10]
-    }.should raise_error(ArgumentError)
+    end
 
-    lambda {
+    assert_raise ArgumentError do
       utf8.chars.to_a
-    }.should raise_error(ArgumentError)
+    end
   end
 
-  it "should extend String, adding an as_utf8 method that returns a String::UTF8 instance" do
-    "".should respond_to(:as_utf8)
-    "".as_utf8.class.should eql(String::UTF8)
+  test "should extend String, adding an as_utf8 method that returns a String::UTF8 instance" do
+    assert "".respond_to?(:as_utf8)
+    assert_equal String::UTF8, "".as_utf8.class
   end
 
-  it "should allow access to the underlying raw string" do
+  test "should allow access to the underlying raw string" do
     raw = @utf8.as_raw
-    raw.class.should eql(String)
+    assert_equal String, raw.class
     if defined? Encoding
-      raw.length.should eql(@utf8_len)
+      assert_equal @utf8_len, raw.length
     else
-      raw.length.should eql(@str.size)
+      assert_equal @str.size, raw.length
     end
   end
 
-  it "should wrap all returned strings to be utf8-aware" do
-    @utf8[0].class.should eql(String::UTF8)
-    @utf8.chars.to_a[0].class.should eql(String::UTF8)
+  test "should wrap all returned strings to be utf8-aware" do
+    assert_equal String::UTF8, @utf8[0].class
+    assert_equal String::UTF8, @utf8.chars.to_a[0].class
   end
 
-  it "clean should replace invalid utf8 chars with '?'" do
+  test "clean should replace invalid utf8 chars with '?'" do
     orig = "provided by Cristian Rodr\355guez."
     clean = "provided by Cristian Rodr?guez."
-    orig.as_utf8.clean.should eql(clean)
+    assert_equal clean, orig.as_utf8.clean
+    assert_equal "asdf24??asdf24", "asdf24\206\222asdf24".as_utf8.clean
+    assert_equal "asdf24?asdf24", "asdf24\342asdf24".as_utf8.clean
+    assert_equal "asdf24??asdf24", "asdf24\342\206asdf24".as_utf8.clean
+    assert_equal "asdf24?asdf24", "asdf24\222asdf24".as_utf8.clean
   end
 
-  it "clean should not replace valid utf8 chars with '?'" do
-    '→'.as_utf8.clean.should eql('→')
+  test "clean should not replace valid utf8 chars with '?'" do
+    assert_equal "asdf24\342\206\222asdf24", "asdf24\342\206\222asdf24".as_utf8.clean
   end
 
   context "#length and #size" do
-    it "should be utf8-aware" do
-      @utf8.length.should eql(@utf8_len)
-      @utf8.size.should eql(@utf8_len)
+    test "should be utf8-aware" do
+      assert_equal @utf8_len, @utf8.length
+      assert_equal @utf8_len, @utf8.size
     end
   end
 
   context "#chars and #each_char" do
-    it "should be utf8-aware" do
+    test "should be utf8-aware" do
       klass = begin
         if defined? Encoding
           Enumerator
@@ -77,19 +81,19 @@ describe String::UTF8 do
         end
       end
 
-      @utf8.chars.class.should eql(klass)
+      assert_equal klass, @utf8.chars.class
       @utf8.chars do |char|
-        char.should_not be_nil
+        assert !char.nil?
       end
       joined = @utf8.chars.to_a.join
-      @utf8.should eql(joined)
-      @utf8.chars.to_a.size.should eql(@utf8_len)
-      @utf8.chars.to_a.should eql(@char_array)
+      assert_equal joined, @utf8
+      assert_equal @utf8_len, @utf8.chars.to_a.size
+      assert_equal @char_array, @utf8.chars.to_a
     end
   end
 
   context "#codepoints and #each_codepoint" do
-    it "should be utf8-aware" do
+    test "should be utf8-aware" do
       klass = begin
         if defined? Encoding
           Enumerator
@@ -98,124 +102,118 @@ describe String::UTF8 do
         end
       end
 
-      @utf8.codepoints.class.should eql(klass)
+      assert_equal klass, @utf8.codepoints.class
       @utf8.codepoints do |codepoint|
-        codepoint.should_not be_nil
+        assert !codepoint.nil?
       end
-      @utf8.codepoints.to_a.size.should eql(@codepoints.size)
-      @utf8.codepoints.to_a.should eql(@codepoints)
+      assert_equal @codepoints.size, @utf8.codepoints.to_a.size
+      assert_equal @codepoints, @utf8.codepoints.to_a
     end
   end
 
   context "[offset] syntax" do
-    it "should be utf8-aware" do
+    test "should be utf8-aware" do
       @char_array.each_with_index do |char, i|
         utf8_char = @utf8[i]
-        utf8_char.should eql(char)
+        assert_equal char, utf8_char
       end
     end
 
-    it "should support negative indices" do
+    test "should support negative indices" do
       utf8_char = @utf8[-5]
-      utf8_char.should eql(@char_array[-5])
+      assert_equal @char_array[-5], utf8_char
     end
 
-    it "should return nil for out of range indices" do
-      @utf8[100].should be_nil
-      @utf8[-100].should be_nil
+    test "should return nil for out of range indices" do
+      assert @utf8[100].nil?
+      assert @utf8[-100].nil?
     end
   end
 
   context "[offset, length] syntax" do
-    it "should be utf8-aware" do
-      utf8_char = @utf8[1, 4]
-      utf8_char.should eql(@char_array[1, 4].join)
-
-      utf8_char = @utf8[0, 6]
-      utf8_char.should eql(@char_array[0, 6].join)
+    test "should be utf8-aware" do
+      assert_equal @char_array[1, 4].join, @utf8[1, 4]
+      assert_equal @char_array[0, 6].join, @utf8[0, 6]
 
       # this will fail due to a bug in 1.9
       unless defined? Encoding
-        utf8_char = @utf8[6, 100]
-        utf8_char.should eql(@char_array[6, 100].join)
+        assert_equal @char_array[6, 100].join, @utf8[6, 100]
       end
 
-      utf8_char = @utf8[-1, 2]
-      utf8_char.should eql(@char_array[-1, 2].join)
-
-      utf8_char = @utf8[-1, 100]
-      utf8_char.should eql(@char_array[-1, 100].join)
-
-      utf8_char = @utf8[0, 0]
-      utf8_char.should eql(@char_array[0, 0].join)
+      assert_equal @char_array[-1, 2].join, @utf8[-1, 2]
+      assert_equal @char_array[-1, 100].join, @utf8[-1, 100]
+      assert_equal @char_array[0, 0].join, @utf8[0, 0]
     end
 
-    it "should return nil for an out of range offset or length" do
-      @utf8[100, 100].should be_nil
-      @utf8[-100, 100].should be_nil
-      @utf8[0, -100].should be_nil
+    test "should return nil for an out of range offset or length" do
+      assert @utf8[100, 100].nil?
+      assert @utf8[-100, 100].nil?
+      assert @utf8[0, -100].nil?
     end
   end
 
   context "[Range] syntax" do
-    it "should be utf8-aware" do
-      utf8_char = @utf8[1..4]
-      utf8_char.should eql(@char_array[1..4].join)
-
-      utf8_char = @utf8[0..6]
-      utf8_char.should eql(@char_array[0..6].join)
+    test "should be utf8-aware" do
+      assert_equal @char_array[1..4].join, @utf8[1..4]
+      assert_equal @char_array[0..6].join, @utf8[0..6]
 
       # this will fail due to a bug in 1.9
       unless defined? Encoding
-        utf8_char = @utf8[6..100]
-        utf8_char.should eql(@char_array[6..100].join)
+        assert_equal @char_array[6..100].join, @utf8[6..100]
       end
 
-      utf8_char = @utf8[-1..2]
-      utf8_char.should eql(@char_array[-1..2].join)
-
-      utf8_char = @utf8[-1..100]
-      utf8_char.should eql(@char_array[-1..100].join)
+      assert_equal @char_array[-1..2].join, @utf8[-1..2]
+      assert_equal @char_array[-1..100].join, @utf8[-1..100]
     end
 
-    it "should return nil for an out of range offset or length" do
-      @utf8[100..100].should be_nil
-      @utf8[-100..100].should be_nil
-      @utf8[0..-100].should eql("")
+    test "should return nil for an out of range offset or length" do
+      assert @utf8[100..100].nil?
+      assert @utf8[-100..100].nil?
+      assert_equal "", @utf8[0..-100]
     end
   end
 
   context "#valid?" do
-    it "should test validity" do
+    test "should test validity" do
       # lets cut right into the middle of a sequence so we know it's bad
       @str.force_encoding('binary') if @str.respond_to?(:force_encoding)
       utf8 = @str[0..1]
       utf8.force_encoding('utf-8') if utf8.respond_to?(:force_encoding)
       utf8 = utf8.as_utf8
 
-      utf8.valid?.should be_false
-      @utf8.valid?.should be_true
+      assert !utf8.valid?
+      assert @utf8.valid?
 
-      "provided by Cristian Rodr\355guez.".as_utf8.should_not be_valid
+      assert !"provided by Cristian Rodr\355guez.".as_utf8.valid?
     end
 
-    it "should test validity using a maximum codepoint" do
+    test "should test validity using a maximum codepoint" do
       highest_codepoint = @utf8.codepoints.to_a.max
 
-      @utf8.valid?(highest_codepoint).should be_true
-      @utf8.valid?(highest_codepoint-1).should be_false
+      assert @utf8.valid?(highest_codepoint)
+      assert !@utf8.valid?(highest_codepoint-1)
     end
   end
 
-  it "[Regexp] syntax shouldn't be supported yet" do
-    lambda {
+  test "[Regexp] syntax shouldn't be supported yet" do
+    assert_raise ArgumentError do
       @utf8[/a/]
-    }.should raise_error(ArgumentError)
+    end
   end
 
-  it "[Regexp, match_index] syntax shouldn't be supported yet" do
-    lambda {
+  test "[Regexp, match_index] syntax shouldn't be supported yet" do
+    assert_raise ArgumentError do
       @utf8[/(a)/, 1]
-    }.should raise_error(ArgumentError)
+    end
+  end
+
+  context "#ascii_only" do
+    test "should return true for a string within the low ascii range" do
+      assert "asdf".as_utf8.ascii_only?
+    end
+
+    test "should return false for a string within the low ascii range" do
+      assert !@char_array.first.as_utf8.ascii_only?
+    end
   end
 end

data/utf8.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.files = `git ls-files`.split("\n")
   s.homepage = %q{http://github.com/brianmario/utf8}
   s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib", "ext"]
+  s.require_paths = ["lib"]
   s.rubygems_version = %q{1.4.2}
   s.summary = %q{A lightweight UTF8-aware String class meant for use with Ruby 1.8}
   s.test_files = `git ls-files spec`.split("\n")

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: utf8
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: 
   segments: 
   - 0
   - 1
-  - 6
-  version: 0.1.6
+  - 7
+  version: 0.1.7
 platform: ruby
 authors: 
 - Brian Lopez
@@ -15,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-25 00:00:00 -04:00
-default_executable: 
+date: 2011-11-22 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake-compiler
@@ -75,6 +74,7 @@ extra_rdoc_files:
 files: 
 - .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
 - MIT-LICENSE
 - README.rdoc
@@ -98,7 +98,6 @@ files:
 - spec/string_scanner_spec.rb
 - spec/string_spec.rb
 - utf8.gemspec
-has_rdoc: true
 homepage: http://github.com/brianmario/utf8
 licenses: []
 
@@ -107,7 +106,6 @@ rdoc_options:
 - --charset=UTF-8
 require_paths: 
 - lib
-- ext
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
@@ -129,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: A lightweight UTF8-aware String class meant for use with Ruby 1.8