coder 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,16 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - rbx-18mode
+  - rbx-19mode
+  - jruby-18mode
+  - jruby-19mode
+  - jruby-head
+  - ruby-head
+before_install:
+  - export JRUBY_OPTS="--server -Xcext.enabled=false -Xcompile.invokedynamic=false"
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-head

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in coder.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Konstantin Haase
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Coder
+
+A Ruby library to deal with encodings, no matter if you're on 1.8 or 1.9, JRuby
+or Rubinius, if you have a working iconv or not, it chooses the best way for you
+to handle String encodings.
+
+## Usage
+
+At the moment, Coder only cleans strings for you. I plan to add string
+conversion and encoding detection later.
+
+### Cleaning Strings
+
+``` ruby
+clean_string = Coder.clean(dirty_string)
+```
+
+You can also specify the encoding:
+
+
+``` ruby
+clean_string = Coder.clean(dirty_string, 'UTF-8')
+```
+
+You can also modify a string in-place:
+
+``` ruby
+Coder.clean! some_string
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+desc "run specs"
+task(:default) { ruby '-S rspec spec' }

data/coder.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'coder/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "coder"
+  gem.version       = Coder::VERSION
+  gem.authors       = ["Konstantin Haase"]
+  gem.email         = ["konstantin.mailinglists@googlemail.com"]
+  gem.description   = %q{handle encodings, no matter what}
+  gem.summary       = %q{library to handle encodings}
+  gem.homepage      = "http://github.com/rkh/coder"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency("rspec", "~> 2.11")
+  gem.add_development_dependency("rake")
+end

data/lib/coder.rb

@@ -0,0 +1,14 @@
+require 'coder/version'
+require 'coder/cleaner'
+
+module Coder
+  extend self
+
+  def clean(str, encoding = nil)
+    Cleaner.new(encoding || 'UTF-8').clean(str)
+  end
+
+  def clean!(str, encoding = nil)
+    str.replace clean(str, encoding)
+  end
+end

data/lib/coder/cleaner.rb

@@ -0,0 +1,13 @@
+require 'coder/cleaner/builtin'
+require 'coder/cleaner/iconv'
+require 'coder/cleaner/simple'
+
+module Coder
+  module Cleaner
+    Default = [Builtin, Iconv, Simple].detect { |c| c.available? }
+
+    def self.new(encoding)
+      Default.new(encoding)
+    end
+  end
+end

data/lib/coder/cleaner/builtin.rb

@@ -0,0 +1,43 @@
+require 'coder/error'
+
+module Coder
+  module Cleaner
+    class Builtin
+      OPTIONS = { :undef => :replace, :invalid => :replace, :replace => "" }
+
+      def self.available?
+        defined? Encoding.find          and
+        defined? EncodingError          and
+        String.method_defined? :encode  and
+        String.method_defined? :force_encoding
+      end
+
+      def initialize(encoding)
+        @encoding = encoding.to_s.upcase
+        @dummy    = @encoding == 'UTF-8' ? 'UTF-16BE' : 'UTF-8' if needs_dummy?
+        @dummy  ||= @encoding
+
+        check_encoding
+      end
+
+      def clean(str)
+        str = str.dup.force_encoding(@encoding)
+        str.encode(@dummy, OPTIONS).encode(@encoding).gsub("\0", "")
+      rescue EncodingError => e
+        raise Coder::Error, e.message
+      end
+
+      private
+
+      def check_encoding
+        Encoding.find(@encoding)
+      rescue ArgumentError => e
+        raise Coder::InvalidEncoding, e.message
+      end
+
+      def needs_dummy?
+        RUBY_VERSION < '2.0'
+      end
+    end
+  end
+end

data/lib/coder/cleaner/iconv.rb

@@ -0,0 +1,40 @@
+require 'coder/error'
+require 'stringio'
+
+module Coder
+  module Cleaner
+    class Iconv
+      def self.load_iconv
+        return if defined? ::Iconv
+        stderr_was, $stderr = $stderr, StringIO.new
+        require 'iconv'
+      ensure
+        $stderr = stderr_was if stderr_was
+      end
+
+      def self.new(*)
+        load_iconv
+        super
+      end
+
+      def self.available?
+        load_iconv
+        !!::Iconv.conv("iso-8859-1//ignore", "utf-8", "\305\253" + "a"*8160)
+      rescue Exception => e
+        false
+      end
+
+      def initialize(encoding)
+        @iconv = ::Iconv.new("#{encoding}//ignore", encoding.to_s)
+      rescue ::Iconv::InvalidEncoding => e
+        raise Coder::InvalidEncoding, e.message
+      end
+
+      def clean(str)
+        @iconv.iconv(str).gsub("\0", "")
+      rescue ::Iconv::Failure => e
+        raise Coder::Error, e.message
+      end
+    end
+  end
+end

data/lib/coder/cleaner/simple.rb

@@ -0,0 +1,32 @@
+require 'coder/error'
+require 'coder/cleaner/simple/byte_buffer'
+require 'coder/cleaner/simple/encodings'
+
+module Coder
+  module Cleaner
+    class Simple
+      def self.available?
+        true
+      end
+
+      def initialize(encoding)
+        const_name = encoding.to_s.upcase.gsub('-', '_')
+        raise Coder::InvalidEncoding, "unknown encoding name - #{encoding}" unless Encodings.const_defined? const_name
+        @encoding, @name = Encodings.const_get(const_name), encoding
+      end
+
+      def clean(str)
+        bytes = ByteBuffer.new(@encoding)
+        str.each_byte { |b| bytes << b }
+        force_encoding bytes.to_s
+      end
+
+      private
+
+      def force_encoding(str)
+        return str unless str.respond_to? :force_encoding
+        str.force_encoding(@name)
+      end
+    end
+  end
+end

data/lib/coder/cleaner/simple/byte_buffer.rb

@@ -0,0 +1,53 @@
+module Coder
+  module Cleaner
+    class Simple
+      class ByteBuffer
+        attr_accessor :encoding, :bytes, :buffer, :outstanding
+
+        def initialize(encoding)
+          @encoding, @bytes = encoding, []
+          clear_buffer
+        end
+
+        def <<(byte)
+          if encoding.garbage? byte, buffer
+            clear_buffer
+          elsif encoding.single_byte? byte, buffer
+            add(byte)
+          elsif encoding.multibyte? byte, buffer
+            fill_buffer(byte)
+          elsif encoding.multibyte_start? byte, buffer
+            start_buffer(byte, encoding.multibyte_size(byte, buffer))
+          else
+            clear_buffer
+          end
+        end
+
+        def to_s
+          bytes.pack('C*')
+        end
+
+        private
+
+        def clear_buffer
+          start_buffer(nil, 0)
+        end
+
+        def start_buffer(byte, size)
+          @buffer, @outstanding = Array(byte), size
+        end
+
+        def fill_buffer(byte)
+          buffer << byte
+          add(buffer)  if buffer.size == outstanding
+          clear_buffer if buffer.size > outstanding
+        end
+
+        def add(input)
+          clear_buffer
+          bytes.concat Array(input)
+        end
+      end
+    end
+  end
+end

data/lib/coder/cleaner/simple/encodings.rb

@@ -0,0 +1,82 @@
+module Coder
+  module Cleaner
+    class Simple
+      module Encodings
+        # Note: This currently does not remove most overlong forms
+        module UTF_8
+          extend self
+
+          def garbage?(input, buffered)
+            return true if input > 244 or input == 192 or input == 193
+            case buffered <=> [244, 143, 191]
+            when -1 then false
+            when  0 then input < 192
+            when  1 then true
+            end
+          end
+
+          def single_byte?(input, buffered)
+            input.between? 1, 127
+          end
+
+          def multibyte_start?(input, buffered)
+            input.between? 192, 244
+          end
+
+          def multibyte?(input, buffered)
+            input.between? 128, 191
+          end
+
+          def multibyte_size(input, buffered)
+            case input
+            when 192..223 then 2
+            when 224..239 then 3
+            when 240..247 then 4
+            when 248..244 then 5
+            when 001..127 then 1
+            else 0
+            end
+          end
+        end
+
+        module UCS_2
+          extend self
+
+          def garbage?(input, buffered)
+            return false unless buffered.size + 1 == multibyte_size
+            input == 0 and buffered.all? { |b| b == 0 }
+          end
+
+          def single_byte?(input, buffered)
+            false
+          end
+
+          def multibyte_start?(input, buffered)
+            buffered.size % multibyte_size == 0
+          end
+
+          def multibyte?(input, buffered)
+            not multibyte_start? input, buffered
+          end
+
+          def multibyte_size(*)
+            2
+          end
+        end
+
+        module UCS_4
+          include UCS_2
+          extend self
+
+          def garbage?(input, buffered)
+            super or input > 0x10FFFF
+          end
+
+          def multibyte_size(*)
+            4
+          end
+        end
+      end
+    end
+  end
+end

data/lib/coder/error.rb

@@ -0,0 +1,7 @@
+module Cleaner
+  class Error < StandardError
+  end
+
+  class InvalidEncoding < Error
+  end
+end

data/lib/coder/version.rb

@@ -0,0 +1,3 @@
+module Coder
+  VERSION = "0.0.1"
+end

data/spec/coder/cleaner_spec.rb

@@ -0,0 +1,33 @@
+require 'coder/cleaner'
+
+shared_examples Coder::Cleaner do
+  let(:encoding) { example.example_group.description }
+  subject { described_class.new(encoding) }
+
+  def self.cleans(from, to = from)
+    it "cleans #{from.inspect} to #{to.inspect}" do
+      subject.clean(from).should be == to
+    end
+  end
+
+  context "UTF-8" do
+    cleans "foo"
+    cleans ""
+    cleans "yummy 🍔 "
+
+    cleans "{foo \xC3 'bar'}", "{foo  'bar'}"
+    cleans "yummy\xE2 \xF0\x9F\x8D\x94 \x9F\x8D\x94", "yummy 🍔 "
+  end
+end
+
+describe Coder::Cleaner::Builtin do
+  it_behaves_like Coder::Cleaner if described_class.available?
+end
+
+describe Coder::Cleaner::Iconv do
+  it_behaves_like Coder::Cleaner if described_class.available?
+end
+
+describe Coder::Cleaner::Simple do
+  it_behaves_like Coder::Cleaner if described_class.available?
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: coder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Konstantin Haase
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: handle encodings, no matter what
+email:
+- konstantin.mailinglists@googlemail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- coder.gemspec
+- lib/coder.rb
+- lib/coder/cleaner.rb
+- lib/coder/cleaner/builtin.rb
+- lib/coder/cleaner/iconv.rb
+- lib/coder/cleaner/simple.rb
+- lib/coder/cleaner/simple/byte_buffer.rb
+- lib/coder/cleaner/simple/encodings.rb
+- lib/coder/error.rb
+- lib/coder/version.rb
+- spec/coder/cleaner_spec.rb
+homepage: http://github.com/rkh/coder
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: library to handle encodings
+test_files:
+- spec/coder/cleaner_spec.rb
+has_rdoc: