utf8 0.1.0

data/lib/utf8.rb

@@ -0,0 +1,5 @@
+require 'utf8/utf8'
+require 'utf8/string'
+
+# explicitly require this in your app if you want to use it
+# require 'utf8/string_scanner'

data/lib/utf8/string.rb

@@ -0,0 +1,19 @@
+class String
+  # Wraps your string in an UTF8-aware version of String
+  def as_utf8
+    String::UTF8.new(self)
+  end
+
+  class UTF8
+    VERSION = "0.1.0"
+
+    # Gives you access to the raw non-UTF8-aware version of the string
+    def as_raw
+      ::String.new(self)
+    end
+
+    alias :size  :length
+    alias :chars :each_char
+    alias :slice :[]
+  end
+end

data/lib/utf8/string_scanner.rb

@@ -0,0 +1,21 @@
+require 'utf8'
+
+class StringScanner
+  # Returns an UTF8-aware version of StringScanner wrapping your original string
+  #
+  # NOTE: this will lose all state associated with the current StringScanner instance
+  # (like the current scan position)
+  def as_utf8
+    StringScanner::UTF8.new(self.string.as_utf8)
+  end
+
+  class UTF8
+    # Returns a non-UTF8-aware version of StringScanner wrapping your original string
+    #
+    # NOTE: this will lose all state associated with the current StringScanner::UTF8 instance
+    # (like the current scan position)
+    def as_raw
+      StringScanner.new(self.string)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'utf8'
+require 'utf8/string_scanner'
+
+require 'rspec' unless defined? Rspec

data/spec/string_scanner_spec.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require File.expand_path('../spec_helper', __FILE__)
+
+describe StringScanner::UTF8 do
+  before(:all) do
+    @char_array = ["怎", "麼", "也", "沒", "人", "寫", "了", "這", "個", "嗎"]
+    @scanner = StringScanner.new(@char_array.join)
+    @utf8_scanner = @scanner.as_utf8
+  end
+
+  it "should extend StringScanner, adding an as_utf8 method that returns a StringScanner::UTF8 instance" do
+    @scanner.should respond_to(:as_utf8)
+    @scanner.as_utf8.class.should eql(StringScanner::UTF8)
+  end
+
+  it "should allow access to a regular (non-utf8-aware) StringScanner based on it's string" do
+    raw = @utf8_scanner.as_raw
+    raw.class.should eql(StringScanner)
+    raw.string.should eql(@utf8_scanner.string)
+  end
+
+  it "#getch should be utf8-aware" do
+    i=0
+    while char = @utf8_scanner.getch
+      char.should eql(@char_array[i])
+      i+=1
+    end
+  end
+
+  it "should be able to be reset" do
+    i=0
+    while char = @utf8_scanner.getch
+      char.should eql(@char_array[i])
+      if i == 4
+        break
+      end
+      i+=1
+    end
+
+    @utf8_scanner.reset
+
+    i=0
+    while char = @utf8_scanner.getch
+      char.should eql(@char_array[i])
+      i+=1
+    end
+  end
+end

data/spec/string_spec.rb

@@ -0,0 +1,151 @@
+# encoding: utf-8
+require File.expand_path('../spec_helper', __FILE__)
+
+describe String::UTF8 do
+  before(:all) do
+    @char_array = ["怎", "麼", "也", "沒", "人", "寫", "了", "這", "個", "嗎"]
+    @str = @char_array.join
+    @utf8 = @str.as_utf8
+    @utf8_len = @char_array.size
+  end
+
+  it "should extend String, adding an as_utf8 method that returns a String::UTF8 instance" do
+    "".should respond_to(:as_utf8)
+    "".as_utf8.class.should eql(String::UTF8)
+  end
+
+  it "should allow access to the underlying raw string" do
+    raw = @utf8.as_raw
+    raw.class.should eql(String)
+    if defined? Encoding
+      raw.length.should eql(@utf8_len)
+    else
+      raw.length.should eql(@str.size)
+    end
+  end
+
+  it "should wrap all returned strings to be utf8-aware" do
+    @utf8[0].class.should eql(String::UTF8)
+    @utf8.chars.to_a[0].class.should eql(String::UTF8)
+  end
+
+  context "#length and #size" do
+    it "should be utf8-aware" do
+      @utf8.length.should eql(@utf8_len)
+      @utf8.size.should eql(@utf8_len)
+    end
+  end
+
+  context "#chars and #each_char" do
+    it "should be utf8-aware" do
+      klass = begin
+        if defined? Encoding
+          Enumerator
+        else
+          Enumerable::Enumerator
+        end
+      end
+
+      @utf8.chars.class.should eql(klass)
+      i=0
+      @utf8.chars do |char|
+        char.should_not be_nil
+        i+=1
+      end
+      joined = @utf8.chars.to_a.join
+      @utf8.should eql(joined)
+      @utf8.chars.to_a.size.should eql(@utf8_len)
+      @utf8.chars.to_a.should eql(@char_array)
+    end
+  end
+
+  context "[offset] syntax" do
+    it "should be utf8-aware" do
+      @char_array.each_with_index do |char, i|
+        utf8_char = @utf8[i]
+        utf8_char.should eql(char)
+      end
+    end
+
+    it "should support negative indices" do
+      utf8_char = @utf8[-5]
+      utf8_char.should eql(@char_array[-5])
+    end
+
+    it "should return nil for out of range indices" do
+      @utf8[100].should be_nil
+      @utf8[-100].should be_nil
+    end
+  end
+
+  context "[offset, length] syntax" do
+    it "should be utf8-aware" do
+      utf8_char = @utf8[1, 4]
+      utf8_char.should eql(@char_array[1, 4].join)
+
+      utf8_char = @utf8[0, 6]
+      utf8_char.should eql(@char_array[0, 6].join)
+
+      # this will fail due to a bug in 1.9
+      unless defined? Encoding
+        utf8_char = @utf8[6, 100]
+        utf8_char.should eql(@char_array[6, 100].join)
+      end
+
+      utf8_char = @utf8[-1, 2]
+      utf8_char.should eql(@char_array[-1, 2].join)
+
+      utf8_char = @utf8[-1, 100]
+      utf8_char.should eql(@char_array[-1, 100].join)
+
+      utf8_char = @utf8[0, 0]
+      utf8_char.should eql(@char_array[0, 0].join)
+    end
+
+    it "should return nil for an out of range offset or length" do
+      @utf8[100, 100].should be_nil
+      @utf8[-100, 100].should be_nil
+      @utf8[0, -100].should be_nil
+    end
+  end
+
+  context "[Range] syntax" do
+    it "should be utf8-aware" do
+      utf8_char = @utf8[1..4]
+      utf8_char.should eql(@char_array[1..4].join)
+
+      utf8_char = @utf8[0..6]
+      utf8_char.should eql(@char_array[0..6].join)
+
+      # this will fail due to a bug in 1.9
+      unless defined? Encoding
+        utf8_char = @utf8[6..100]
+        utf8_char.should eql(@char_array[6..100].join)
+      end
+
+      utf8_char = @utf8[-1..2]
+      utf8_char.should eql(@char_array[-1..2].join)
+
+      utf8_char = @utf8[-1..100]
+      utf8_char.should eql(@char_array[-1..100].join)
+    end
+
+    it "should return nil for an out of range offset or length" do
+      @utf8[100..100].should be_nil
+      @utf8[-100..100].should be_nil
+      @utf8[0..-100].should eql("")
+    end
+  end
+
+  it "[Regexp] syntax shouldn't be supported yet" do
+    lambda {
+      @utf8[/a/]
+    }.should raise_error(ArgumentError)
+  end
+
+  it "[Regexp, match_index] syntax shouldn't be supported yet" do
+    lambda {
+      @utf8[/(a)/, 1]
+    }.should raise_error(ArgumentError)
+  end
+end

data/utf8.gemspec

@@ -0,0 +1,37 @@
+# encoding: utf-8
+
+Gem::Specification.new do |s|
+  s.name = %q{utf8}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Brian Lopez"]
+  s.date = %q{2011-01-12}
+  s.email = %q{seniorlopez@gmail.com}
+  s.extensions = ["ext/utf8/extconf.rb"]
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [".gitignore", "MIT-LICENSE", "README.rdoc", "Rakefile", "benchmark/active_support.rb", "benchmark/test.txt", "ext/utf8/ext.c", "ext/utf8/ext.h", "ext/utf8/extconf.rb", "ext/utf8/string_scanner_utf8.c", "ext/utf8/string_scanner_utf8.h", "ext/utf8/string_utf8.c", "ext/utf8/string_utf8.h", "ext/utf8/utf8.c", "ext/utf8/utf8.h", "lib/utf8.rb", "lib/utf8/string.rb", "lib/utf8/string_scanner.rb", "spec/spec_helper.rb", "spec/string_scanner_spec.rb", "spec/string_spec.rb", "utf8.gemspec"]
+  s.homepage = %q{http://github.com/brianmario/utf8}
+  s.require_paths = ["lib", "ext"]
+  s.rubygems_version = %q{1.4.2}
+  s.summary = %q{A lightweight UTF8-aware String class meant for use with Ruby 1.8}
+  s.test_files = ["spec/spec_helper.rb", "spec/string_scanner_spec.rb", "spec/string_spec.rb"]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rake-compiler>, [">= 0.7.5"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+    else
+      s.add_dependency(%q<rake-compiler>, [">= 0.7.5"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<rake-compiler>, [">= 0.7.5"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+  end
+end
+

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification 
+name: utf8
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Brian Lopez
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-12 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake-compiler
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 7
+        - 5
+        version: 0.7.5
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: 
+email: seniorlopez@gmail.com
+executables: []
+
+extensions: 
+- ext/utf8/extconf.rb
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .gitignore
+- MIT-LICENSE
+- README.rdoc
+- Rakefile
+- benchmark/active_support.rb
+- benchmark/test.txt
+- ext/utf8/ext.c
+- ext/utf8/ext.h
+- ext/utf8/extconf.rb
+- ext/utf8/string_scanner_utf8.c
+- ext/utf8/string_scanner_utf8.h
+- ext/utf8/string_utf8.c
+- ext/utf8/string_utf8.h
+- ext/utf8/utf8.c
+- ext/utf8/utf8.h
+- lib/utf8.rb
+- lib/utf8/string.rb
+- lib/utf8/string_scanner.rb
+- spec/spec_helper.rb
+- spec/string_scanner_spec.rb
+- spec/string_spec.rb
+- utf8.gemspec
+has_rdoc: true
+homepage: http://github.com/brianmario/utf8
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: A lightweight UTF8-aware String class meant for use with Ruby 1.8
+test_files: 
+- spec/spec_helper.rb
+- spec/string_scanner_spec.rb
+- spec/string_spec.rb