coder 0.1.2 → 0.2.0

data/bench.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+
+# JRuby 1.7.0-preview2
+#                              user     system      total        real
+# Coder::Cleaner::Java     0.360000   0.010000   0.370000 (  0.121000)
+# Coder::Cleaner::Iconv    0.290000   0.010000   0.300000 (  0.103000)
+# Coder::Cleaner::Simple   1.010000   0.020000   1.030000 (  0.367000)
+
+# MRI 1.9.3
+#                               user     system      total        real
+# Coder::Cleaner::Builtin   0.060000   0.000000   0.060000 (  0.057767)
+# Coder::Cleaner::Iconv     0.020000   0.000000   0.020000 (  0.022351)
+# Coder::Cleaner::Simple    0.480000   0.000000   0.480000 (  0.486451)
+
+require 'benchmark'
+require 'coder'
+
+strings = [
+  "yummy\xE2 \xF0\x9F\x8D\x94 \x9F\x8D\x94",
+  "{foo \xC3 'bar'}",
+  "yummy 🍔 " * 10
+]
+
+Benchmark.bmbm do |x|
+  Coder::Cleaner::AVAILABLE.each do |cleaner|
+    x.report cleaner.to_s do
+      1000.times do
+        strings.each do |str|
+          cleaner.new('UTF-8').clean(str)
+        end
+      end
+    end
+  end
+end

data/lib/coder/cleaner.rb

@@ -5,10 +5,21 @@ require 'coder/cleaner/simple'
 
 module Coder
   module Cleaner
-    Default = [Builtin, Java, Iconv, Simple].detect { |c| c.available? }
+    ALL = [ Builtin, Java, Iconv, Simple ]
+    AVAILABLE = ALL.select { |e| e.available? }
+
+    def self.available?
+      AVAILABLE.any?
+    end
+
+    def self.supports?(encoding)
+      AVAILABLE.any? { |e| e.supports? encoding }
+    end
 
     def self.new(encoding)
-      Default.new(encoding)
+      cleaner = AVAILABLE.detect { |e| e.supports? encoding }
+      raise Coder::InvalidEncoding, "unknown encoding name - #{encoding}" unless cleaner
+      cleaner.new(encoding)
     end
   end
 end

data/lib/coder/cleaner/builtin.rb

@@ -13,6 +13,12 @@ module Coder
         !defined?(RUBY_ENGINE) or RUBY_ENGINE == 'ruby'
       end
 
+      def self.supports?(encoding)
+        Encoding.find(encoding)
+      rescue ArgumentError
+        false
+      end
+
       def self.has_encoding?
         defined? Encoding.find          and
         defined? EncodingError          and
@@ -30,7 +36,7 @@ module Coder
 
       def clean(str)
         str = str.dup.force_encoding(@encoding)
-        str.encode(@dummy, OPTIONS).encode(@encoding).gsub("\0", "")
+        str.encode(@dummy, OPTIONS).encode(@encoding).gsub("\0".encode(@encoding), "")
       rescue EncodingError => e
         raise Coder::Error, e.message
       end
@@ -38,9 +44,8 @@ module Coder
       private
 
       def check_encoding
-        Encoding.find(@encoding)
-      rescue ArgumentError => e
-        raise Coder::InvalidEncoding, e.message
+        return if self.class.supports? @encoding
+        raise Coder::InvalidEncoding, "unknown encoding name - #{@encoding}"
       end
 
       def needs_dummy?

data/lib/coder/cleaner/iconv.rb

@@ -17,6 +17,12 @@ module Coder
         super
       end
 
+      def self.supports?(encoding)
+        encoding.to_s !~ /^ucs/i and ::Iconv.new("#{encoding}//ignore", encoding.to_s)
+      rescue Exception
+        false
+      end
+
       def self.available?
         load_iconv
         !!::Iconv.conv("iso-8859-1//ignore", "utf-8", "\305\253" + "a"*8160)
@@ -25,13 +31,15 @@ module Coder
       end
 
       def initialize(encoding)
-        @iconv = ::Iconv.new("#{encoding}//ignore", encoding.to_s)
+        @nullbyte = "\0"
+        @iconv    = ::Iconv.new("#{encoding}//ignore", encoding.to_s)
+        @nullbyte.encode! encoding if @nullbyte.respond_to? :encode!
       rescue ::Iconv::InvalidEncoding => e
         raise Coder::InvalidEncoding, e.message
       end
 
       def clean(str)
-        @iconv.iconv(str).gsub("\0", "")
+        @iconv.iconv(str).gsub(@nullbyte, "")
       rescue ::Iconv::Failure => e
         raise Coder::Error, e.message
       end

data/lib/coder/cleaner/java.rb

@@ -10,12 +10,18 @@ module Coder
         false
       end
 
+      def self.supports?(encoding)
+        encoding.to_s =~ /^utf-8$/i
+      end
+
       def initialize(encoding)
-        encoding = encoding.to_s.upcase
-        @charset = ::Java::JavaNioCharset::Charset.for_name(encoding)
-        @decoder = @charset.new_decoder
+        encoding  = encoding.to_s.upcase
+        @nullbyte = "\0"
+        @charset  = ::Java::JavaNioCharset::Charset.for_name(encoding)
+        @decoder  = @charset.new_decoder
         @decoder.on_malformed_input(::Java::JavaNioCharset::CodingErrorAction::IGNORE)
         @decoder.on_unmappable_character(::Java::JavaNioCharset::CodingErrorAction::IGNORE)
+        @nullbyte.encode! encoding if @nullbyte.respond_to? :encode!
       rescue ::Java::JavaNioCharset::UnsupportedCharsetException, ::Java::JavaNioCharset::IllegalCharsetNameException
         raise Coder::InvalidEncoding, "unknown encoding name - #{encoding}"
       rescue ::Java::JavaLang::RuntimeException => e
@@ -24,7 +30,7 @@ module Coder
 
       def clean(str)
         buffer = ::Java::JavaNio::ByteBuffer.wrap(str.to_java_bytes)
-        @decoder.decode(buffer).to_s
+        @decoder.decode(buffer).to_s.gsub(@nullbyte, '')
       rescue Java::JavaLang::RuntimeException => e
         raise Coder::Error, e.message, e.backtrace
       end

data/lib/coder/cleaner/simple.rb

@@ -9,9 +9,16 @@ module Coder
         true
       end
 
+      def self.supports?(encoding)
+        const_name = encoding.to_s.upcase.gsub('-', '_')
+        Encodings.const_defined? const_name
+      rescue NameError
+        false
+      end
+
       def initialize(encoding)
         const_name = encoding.to_s.upcase.gsub('-', '_')
-        raise Coder::InvalidEncoding, "unknown encoding name - #{encoding}" unless coding_available? const_name
+        raise Coder::InvalidEncoding, "unknown encoding name - #{encoding}" unless self.class.supports? const_name
         @encoding, @name = Encodings.const_get(const_name), encoding
       end
 
@@ -23,12 +30,6 @@ module Coder
 
       private
 
-      def coding_available?(const_name)
-        Encodings.const_defined? const_name
-      rescue NameError
-        false
-      end
-
       def force_encoding(str)
         return str unless str.respond_to? :force_encoding
         str.force_encoding(@name)

data/lib/coder/cleaner/simple/encodings.rb

@@ -39,11 +39,12 @@ module Coder
           end
         end
 
-        module UCS_2
+        module UCS_2BE
           extend self
 
           def garbage?(input, buffered)
             return false unless buffered.size + 1 == multibyte_size
+            return true if codepoint(buffered + [input]) > 0x10FFFF
             input == 0 and buffered.all? { |b| b == 0 }
           end
 
@@ -62,16 +63,16 @@ module Coder
           def multibyte_size(*)
             2
           end
+
+          def codepoint(values)
+            values.inject { |a,b| (a << 8) + b }
+          end
         end
 
-        module UCS_4
-          include UCS_2
+        module UCS_4BE
+          include UCS_2BE
           extend self
 
-          def garbage?(input, buffered)
-            super or input > 0x10FFFF
-          end
-
           def multibyte_size(*)
             4
           end

data/lib/coder/version.rb

@@ -1,3 +1,3 @@
 module Coder
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/spec/coder/cleaner_spec.rb

@@ -1,43 +1,80 @@
 # encoding: UTF-8
 require 'coder/cleaner'
 require 'coder/error'
+require 'support/clean_helpers'
 
 shared_examples Coder::Cleaner do
-  let(:encoding) { example.example_group.description }
-  subject { described_class.new(encoding) }
-
-  def self.cleans(from, to = from)
-    it "cleans #{from.inspect} to #{to.inspect}" do
-      subject.clean(from).should be == to
-    end
-  end
-
-  context "UTF-8" do
+  encoding "UTF-8" do
     cleans "foo"
     cleans ""
     cleans "yummy 🍔 "
-
+    cleans "\0", ""
     cleans "{foo \xC3 'bar'}", "{foo  'bar'}"
     cleans "yummy\xE2 \xF0\x9F\x8D\x94 \x9F\x8D\x94", "yummy 🍔 "
   end
 
-  context "Unknown Encoding" do
-    it { expect { subject }.to raise_error(Coder::InvalidEncoding) }
+  encoding "UCS-2BE" do
+    cleans "\x00f\x00o\x00o"
+    cleans "\x00f\x00ox", "\x00f\x00o"
+    cleans "\x00f\x00o\x00\x00", "\x00f\x00o"
   end
-end
 
-describe Coder::Cleaner::Builtin do
-  it_behaves_like Coder::Cleaner if described_class.available?
-end
+  encoding "UCS-4BE" do
+    cleans "\x00\x00\x00f\x00\x00\x00o\x00\x00\x00o"
+    cleans "\x00\x00\x00f\x00\x00\x00o\x00\x00x", "\x00\x00\x00f\x00\x00\x00o"
+    cleans "\x00\x00\x00f\x00\x00\x00o\x00\x00\x00\x00", "\x00\x00\x00f\x00\x00\x00o"
+    cleans "\xFF\xFF\x10\x10", ""
+  end
 
-describe Coder::Cleaner::Java do
-  it_behaves_like Coder::Cleaner if described_class.available?
+  context "unknown encoding" do
+    it "raises an exception" do
+      expect { described_class.new('foobar') }.
+        to raise_error(Coder::InvalidEncoding)
+    end
+  end
 end
 
-describe Coder::Cleaner::Iconv do
-  it_behaves_like Coder::Cleaner if described_class.available?
+describe Coder::Cleaner do
+  include CleanHelpers
+  it_behaves_like Coder::Cleaner
+
+  it { should support('UTF-8') }
+  it { should support('UCS-2BE') }
+  it { should support('UCS-4BE') }
 end
 
 describe Coder::Cleaner::Simple do
-  it_behaves_like Coder::Cleaner if described_class.available?
+  include CleanHelpers
+  it_behaves_like Coder::Cleaner
+
+  it { should support('UTF-8') }
+  it { should support('UCS-2BE') }
+  it { should support('UCS-4BE') }
 end
+
+describe Coder::Cleaner::Builtin do
+  include CleanHelpers
+  it_behaves_like Coder::Cleaner
+
+  it { should support('UTF-8') }
+  it { should support('UCS-2BE') }
+  it { should support('UCS-4BE') }
+end if Coder::Cleaner::Builtin.available?
+
+describe Coder::Cleaner::Java do
+  include CleanHelpers
+  it_behaves_like Coder::Cleaner
+
+  it { should support('UTF-8') }
+  it { should_not support('UCS-2BE') }
+  it { should_not support('UCS-4BE') }
+end if Coder::Cleaner::Java.available?
+
+describe Coder::Cleaner::Iconv do
+  include CleanHelpers
+  it_behaves_like Coder::Cleaner
+
+  it { should support('UTF-8') }
+  it { should_not support('UCS-2BE') }
+  it { should_not support('UCS-4BE') }
+end if Coder::Cleaner::Iconv.available?

data/spec/support/clean_helpers.rb

@@ -0,0 +1,33 @@
+module CleanHelpers
+  module ClassMethods
+    def encoding(encoding, &block)
+      return unless described_class.supports? encoding
+      context(encoding) do
+        let(:encoding) { encoding}
+        instance_eval(&block)
+      end
+    end
+
+    def cleans(from, to = from)
+      it "cleans #{from.inspect} to #{to.inspect}" do
+        result = described_class.new(encoding).clean(binary(from))
+        binary(result).should be == binary(to)
+      end
+    end
+  end
+
+  def binary(str)
+    return str unless str.respond_to? :force_encoding
+    str.force_encoding('binary')
+  end
+
+  def support(encoding)
+    be_supports(encoding)
+  end
+
+  def self.append_features(obj)
+    obj.extend ClassMethods
+    obj.subject { obj.described_class }
+    super
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coder
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-23 00:00:00.000000000 Z
+date: 2012-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -56,6 +56,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bench.rb
 - coder.gemspec
 - lib/coder.rb
 - lib/coder/cleaner.rb
@@ -68,6 +69,7 @@ files:
 - lib/coder/error.rb
 - lib/coder/version.rb
 - spec/coder/cleaner_spec.rb
+- spec/support/clean_helpers.rb
 homepage: http://github.com/rkh/coder
 licenses: []
 post_install_message: 
@@ -94,4 +96,5 @@ specification_version: 3
 summary: library to handle encodings
 test_files:
 - spec/coder/cleaner_spec.rb
+- spec/support/clean_helpers.rb
 has_rdoc: