encoding_checker 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--format=documentation

data/.rvmrc

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p125@encoding_checker"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.10.3" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+  if [[ $- == *i* ]] # check for interactive shells
+  then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
+  else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells
+  fi
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create use  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | grep -vE '^Using|Your bundle is complete'
+# fi

data/.travis.yml

@@ -0,0 +1,5 @@
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - rbx-19mode
+  - jruby-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in encoding_checker.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Pavel Forkert
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# EncodingChecker
+
+When you need to parse some text files - you need to be sure, that they are in some particular encoding
+before actually parsing them. For example, some symbols are invalid for UTF-8 encoding, but nevertheless
+files which are mainly in UTF-8 can contain some invalid characters and many of editors will not show you that.
+This gem will help you identify lines and characters of the text which are invalid for particular encoding.
+
+## Installation
+
+This gem relies on encoding information, which is available only in Ruby 1.9.x.
+Maybe sometimes I'll add 1.8.x support through iconv library, but for now 1.8.x is not supported.
+
+Add this line to your application's Gemfile:
+
+  gem 'encoding_checker'
+
+And then execute:
+
+  $ bundle
+
+Or install it yourself as:
+
+  $ gem install encoding_checker
+
+## Usage
+
+  # instantiate checker with encoding name
+  checker = EncodingChecker.new("utf-8")
+  # check any particular text
+  result = checker.check("some string with wrong\xA0symbol")
+
+  unless result.empty?
+    result.invalid_lines.each do |line|
+      # use line.content, line.index and line.invalid_characters
+      line.invalid_characters.each do |character|
+        # use character.content and character.index
+      end
+    end
+  end
+
+  # raises EncodingChecker::Error
+  checker.check!("some string with wrong\xA0symbol")
+
+Read the specs for more information.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+end
+
+task :default => :spec

data/encoding_checker.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/encoding_checker/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Pavel Forkert"]
+  gem.email         = ["fxposter@gmail.com"]
+  gem.description   = %q{This gem will helps you identify lines and characters of the text which are invalid for particular encoding}
+  gem.summary       = %q{When you need to parse some text files - you need to be sure, that they are in some particular encoding before actually parsing them. For example, some symbols are invalid for UTF-8 encoding, but nevertheless files which are mainly in UTF-8 can contain some invalid characters and many of editors will not show you that. This gem will help you identify lines and characters of the text which are invalid for particular encoding.}
+  gem.homepage      = "https://github.com/fxposter/encoding_checker"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "encoding_checker"
+  gem.require_paths = ["lib"]
+  gem.version       = EncodingChecker::VERSION
+
+  gem.add_development_dependency 'rspec', '~> 2.8.0'
+end

data/lib/encoding_checker/version.rb

@@ -0,0 +1,3 @@
+class EncodingChecker
+  VERSION = "0.0.1"
+end

data/lib/encoding_checker.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+require "encoding_checker/version"
+
+# checker = EncodingChecker.new("utf-8")
+# checker.check!("some string with wrong\xA0symbol")
+class EncodingChecker
+  def initialize(encoding)
+    @encoding = encoding
+  end
+
+  def check(string)
+    string = string.dup
+    string.force_encoding(@encoding)
+    return Match.new(@encoding, []) if string.valid_encoding?
+    errors = []
+    string.each_line.with_index do |line, line_index|
+      unless line.valid_encoding?
+        line_errors = []
+        line.each_char.with_index do |char, index|
+          unless char.valid_encoding?
+            line_errors << CharacterMatch.new(index, char)
+          end
+        end
+        errors << LineMatch.new(line_index, line.strip, line_errors)
+      end
+    end
+    Match.new(@encoding, errors)
+  end
+
+  def check!(string)
+    check(string).tap do |match|
+      raise Error, match unless match.empty?
+    end
+  end
+
+  class Match < Struct.new(:encoding, :invalid_lines)
+    def empty?
+      invalid_lines.empty?
+    end
+
+    def to_s
+      invalid_lines.map { |line|
+        result = %(#{encoding.to_s.upcase} error on line #{line}"\n)
+        result << 'Invalid characters: ' << line.invalid_characters.join(', ') << "\n"
+        result
+      }.join("\n")
+    end
+  end
+
+  class LineMatch < Struct.new(:index, :content, :invalid_characters)
+    def to_s
+      %(#{index}: "#{content}")
+    end
+  end
+
+  class CharacterMatch < Struct.new(:index, :content)
+    def to_s
+      "#{content.inspect} (#{index})"
+    end
+  end
+
+  class Error < RuntimeError
+    attr_reader :result
+
+    def initialize(result)
+      @result = result
+      super
+    end
+  end
+end

data/spec/encoding_checker_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe EncodingChecker do
+  let(:checker) { EncodingChecker.new("utf-8") }
+  let(:valid_string) { "some string with only_right symbols" }
+  let(:invalid_symbol) { "\xA0" }
+  let(:invalid_string) { "some string with wrong#{invalid_symbol}symbol" }
+  let(:invalid_text) { [valid_string, invalid_string].join("\n") }
+
+  describe '#check(string)' do
+    it 'returns result which contains invalid lines and characters in them' do
+      result = checker.check(invalid_text)
+      result.should_not be_empty
+
+      result.invalid_lines.should have(1).element
+      invalid_line = result.invalid_lines[0]
+      check_line(invalid_line, invalid_string, 1)
+
+      invalid_line.invalid_characters.should have(1).element
+      invalid_character = invalid_line.invalid_characters[0]
+      check_character(invalid_character, invalid_symbol, 22)
+    end
+
+    it 'returns empty result when a string is in the specified encoding' do
+      checker = EncodingChecker.new("utf-8")
+      result = checker.check(valid_string)
+      result.should be_empty
+      result.invalid_lines.should be_empty
+    end
+  end
+
+  describe '#check!(string)' do
+    it 'raises EncodingChecker::Error when errors are present' do
+      expect {
+        checker.check!(invalid_string)
+      }.to raise_error(EncodingChecker::Error)
+    end
+
+    it 'raises EncodingChecker::Error which contains invalid lines when errors are present' do
+      begin
+        checker.check!(invalid_string)
+      rescue EncodingChecker::Error => e
+        e.result.should_not be_nil
+        e.result.invalid_lines.should have(1).element
+      end
+    end
+
+    it 'returns true when no errors are present' do
+      checker.check!(valid_string).should be_true
+    end
+  end
+
+  def check_line(line, content, index)
+    line.content.should == content
+    line.index.should == index
+  end
+
+  def check_character(character, content, index)
+    character.content.should == content
+    character.index.should == index
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require 'bundler/setup'
+Bundler.require(:default, :development)

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: encoding_checker
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Pavel Forkert
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70149927503160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: *70149927503160
+description: This gem will helps you identify lines and characters of the text which
+  are invalid for particular encoding
+email:
+- fxposter@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .rvmrc
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- encoding_checker.gemspec
+- lib/encoding_checker.rb
+- lib/encoding_checker/version.rb
+- spec/encoding_checker_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/fxposter/encoding_checker
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: When you need to parse some text files - you need to be sure, that they are
+  in some particular encoding before actually parsing them. For example, some symbols
+  are invalid for UTF-8 encoding, but nevertheless files which are mainly in UTF-8
+  can contain some invalid characters and many of editors will not show you that.
+  This gem will help you identify lines and characters of the text which are invalid
+  for particular encoding.
+test_files:
+- spec/encoding_checker_spec.rb
+- spec/spec_helper.rb