utf8_validator 0.0.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.2"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,18 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.8.7)
+    rcov (0.9.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.5.2)
+  rcov

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Guy Allard
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,43 @@
+= A UTF-8 Validator State Machine
+
+Provides an implementation of a state machine for validating UTF-8 encoded strings.  Clients may request that encoding errors be reported in several ways:
+
+* simple true / false indicator
+* a raised exception
+
+== What This gem does Not Provide
+
+* UTF-8 Encoding
+* UTF-8 Decoding
+
+That functionality is left as an exercise for the reader.
+
+== Thanks To
+
+The Unicode Consortium:: At http://unicode.org/ for all the information published there.
+Frank Yung-Fong Tang:: For the state machine algorithm.  See: http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
+
+== A Word On Ruby Versions
+
+It is expected that this validator will be used in Ruby environments prior to 1.9.x. However, nothing prohibits it's use with Ruby 1.9.
+
+== Reporting Issues
+
+Please report issues on the tracker at github.
+
+== Contributing to the utf8_validator gem
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so it does not break in in a future version unintentionally.
+* Please try not to modify the Rakefile or VERSION file. If you require your own version please isolate the version update to its own commit so cherry-pick or rebase can be used to skip it.
+* Request a pull.
+
+== Copyright
+
+Copyright (c) 2011 Guy Allard. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "utf8_validator"
+  gem.homepage = "http://github.com/gmallard/utf8_validator"
+  gem.license = "MIT"
+  gem.summary = %Q{A UTF-8 Encoding Validator.}
+  gem.description = %Q{A State Machine implementation of a UTF-8 Encoding 
+Validation algorithm.}
+  gem.email = "allard.guy.m@gmail.com"
+  gem.authors = ["Guy Allard"]
+
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+
+  gem.add_development_dependency 'bundler', '>= 2.1.2'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "UTF-8 Validator #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/utf8_validator.rb

@@ -0,0 +1,4 @@
+#
+require 'validation/validator'
+require 'validation/errors'
+

data/lib/validation/errors.rb

@@ -0,0 +1,14 @@
+module UTF8
+  #
+  # == Purpose
+  #
+  # General UTF-8 validation error class.  Clients that raise this error
+  # should override the default message.
+  #
+  class ValidationError < ::RuntimeError
+    #
+    def message()
+      "general UTF-8 validation error"
+    end
+  end
+end

data/lib/validation/validator.rb

@@ -0,0 +1,284 @@
+# encoding: utf-8
+#
+=begin
+
+http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
+
+    * state START
+
+          * Input = 0x00-0x7F : change state to START
+          * Input = 0xC2-0xDF: change state to A
+          * Input = 0xE1-0xEC, 0xEE-0xEF: change state to B
+          * Input = 0xE0: change state to C
+          * Input = 0xED: change state to D
+          * Input = 0xF1-0xF3:change state to E
+          * Input = 0xF0: change state to F
+          * Input = 0xF4: change state to G
+          * Input = Others (0x80-0xBF,0xC0-0xC1, 0xF5-0xFF): ERROR
+
+    * state A
+          o Input = 0x80-0xBF: change state to START
+          o Others: ERROR
+    * state B
+          o Input = 0x80-0xBF: change state to A
+          o Others: ERROR
+    * state C
+          o Input = 0xA0-0xBF: change state to A
+          o Others: ERROR
+    * state D
+          o Input = 0x80-0x9F: change state to A
+          o Others: ERROR
+    * state E
+          o Input = 0x80-0xBF: change state to B
+          o Others: ERROR
+    * state F
+          o Input = 0x90-0xBF: change state to B
+          o Others: ERROR
+    * state G
+          o Input = 0x80-0x8F: change state to B
+          o Others: ERROR
+
+This state machine can be easily understood by:
+
+a) examining the machine behavior as documented
+b) reference to an excellent UTF-8 article with accompanying table here:
+
+http://en.wikipedia.org/wiki/UTF-8
+
+=end
+#
+# == Purpose
+#
+# Container for UTF-8 validator.
+#
+module UTF8
+#
+# == Purpose
+#
+# Validate UTF-8 primarily in a Ruby environments other than 1.9.
+#
+# Instances of this class are thread safe, and a single instance may be used
+# safely by multiple concurrent threads, with one caveat:
+#
+# The value of #{Validator::DEBUG} must not be changed by any thread.
+#
+#--
+# Copyright (c) 2011 Guy Allard
+#
+class Validator
+  #
+  # For use during development only.
+  #
+  DEBUG=false
+
+  #
+  # Validate the supplied string for proper UTF-8 encoding.
+  #
+  # Calling Sequence:
+  #
+  #    validator.valid_encoding?(string)                         -> true or false
+  #    validator.valid_encoding?(string, raise_on_error)         -> true or exception
+  #
+  # Parameters:
+  #
+  # string::         the string to validate
+  # raise_on_error:: a flag to indicate failure behavior
+  # 
+  #
+  def valid_encoding?(string, raise_on_error = false)
+    bytes = string.bytes
+    #
+    valid = true
+    index = -1
+    nb_hex = nil
+    ni_hex = nil
+    state = "start"
+    next_byte_save = nil
+    #
+    bytes.each do |next_byte|
+      index += 1
+      next_byte_save = next_byte
+      ni_hex = sprintf "%x", index
+      nb_hex = sprintf "%x", next_byte
+      puts "Top: #{next_byte}(0x#{nb_hex}), index: #{index}(0x#{ni_hex})" if DEBUG
+      case state
+
+        # State: 'start'
+        # The 'start' state:
+        # * handles all occurrences of valid single byte characters i.e., the ASCII character set
+        # * provides state transition logic for start bytes of valid characters with 2-4 bytes
+        # * signals a validation failure for all other single bytes
+        # 
+        when "start"
+          puts "state: start" if DEBUG
+          case next_byte
+
+            # ASCII
+            # * Input = 0x00-0x7F : change state to START
+            when (0x00..0x7f)
+              puts "state: start 1" if DEBUG
+              state = "start"
+
+            # Start byte of two byte characters
+            # * Input = 0xC2-0xDF: change state to A
+            when (0xc2..0xdf)
+              puts "state: start 2" if DEBUG
+              state = "a"
+
+            # Start byte of some three byte characters
+            # * Input = 0xE1-0xEC, 0xEE-0xEF: change state to B
+            when (0xe1..0xec)
+              puts "state: start 3" if DEBUG
+              state = "b"
+            when (0xee..0xef)
+              puts "state: start 4" if DEBUG
+              state = "b"
+
+            # Start byte of special three byte characters
+            # * Input = 0xE0: change state to C
+            when 0xe0
+              puts "state: start 5" if DEBUG
+              state = "c"
+
+            # Start byte of the remaining three byte characters
+            # * Input = 0xED: change state to D
+            when 0xed
+              puts "state: start 6" if DEBUG
+              state = "d"
+
+            # Start byte of some four byte characters
+            # * Input = 0xF1-0xF3:change state to E
+            when (0xf1..0xf3)
+              puts "state: start 7" if DEBUG
+              state = "e"
+
+            # Start byte of special four byte characters
+            # * Input = 0xF0: change state to F
+            when 0xf0
+              puts "state: start 8" if DEBUG
+              state = "f"
+
+            # Start byte of very special four byte characters
+            # * Input = 0xF4: change state to G
+            when 0xf4
+              puts "state: start 9" if DEBUG
+              state = "g"
+
+            # All other single characters are invalid
+            # * Input = Others (0x80-0xBF,0xC0-0xC1, 0xF5-0xFF): ERROR
+            else
+              valid = false
+              break
+          end # of the inner case
+
+        # The last continuation byte of a 2, 3, or 4 byte character
+        # State: 'a'
+        #  o Input = 0x80-0xBF: change state to START
+        #  o Others: ERROR
+        when "a"
+          puts "state: a" if DEBUG
+          if (0x80..0xbf) === next_byte
+            state = "start"
+          else
+            valid = false
+            break
+          end
+
+        # The first continuation byte for most 3 byte characters
+        # State: 'b'
+        # o Input = 0x80-0xBF: change state to A
+        # o Others: ERROR
+        when "b"
+          puts "state: b" if DEBUG
+          if (0x80..0xbf) === next_byte
+            state = "a"
+          else
+            valid = false
+            break
+          end
+
+        # The first continuation byte for some special 3 byte characters
+        # State: 'c'
+        # o Input = 0xA0-0xBF: change state to A
+        # o Others: ERROR
+        when "c"
+          puts "state: c" if DEBUG
+          if (0xa0..0xbf) === next_byte
+            state = "a"
+          else
+            valid = false
+            break
+          end
+
+        # The first continuation byte for the remaining 3 byte characters
+        # State: 'd'
+        # o Input = 0x80-0x9F: change state to A
+        # o Others: ERROR
+        when "d"
+          puts "state: d" if DEBUG
+          if (0x80..0x9f) === next_byte
+            state = "a"
+          else
+            valid = false
+            break
+          end
+
+        # The first continuation byte for some 4 byte characters
+        # State: 'e'
+        # o Input = 0x80-0xBF: change state to B
+        # o Others: ERROR
+        when "e"
+          puts "state: e" if DEBUG
+          if (0x80..0xbf) === next_byte
+            state = "b"
+          else
+            valid = false
+            break
+          end
+
+        # The first continuation byte for some special 4 byte characters
+        # State: 'f'
+        # o Input = 0x90-0xBF: change state to B
+        # o Others: ERROR
+        when "f"
+          puts "state: f" if DEBUG
+          if (0x90..0xbf) === next_byte
+            state = "b"
+          else
+            valid = false
+            break
+          end
+
+        # The first continuation byte for the remaining 4 byte characters
+        # State: 'g'
+        # o Input = 0x80-0x8F: change state to B
+        # o Others: ERROR
+        when "g"
+          puts "state: g" if DEBUG
+          if (0x80..0x8f) === next_byte
+            state = "b"
+          else
+            valid = false
+            break
+          end
+
+        #
+        else
+          raise RuntimeError, "state: default"
+      end
+    end
+    #
+    puts "State at end: #{state}" if DEBUG
+    # Catch truncation at end of string
+    if valid and state != 'start'
+      valid = false
+    end
+    #
+    if !valid and raise_on_error
+      raise ValidationError, "Invalid byte:#{next_byte_save}(0x#{nb_hex}),index:#{index}(0x#{ni_hex})"
+    end
+    #
+    valid
+  end # of valid_encoding?
+end # of class
+end # of module

data/test/helper.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'utf8_validator'
+
+class Test::Unit::TestCase
+end

data/test/test_raise_request.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+#
+require 'helper'
+
+#
+# == Purpose
+#
+# Test raise_on_error functionality.
+#
+class TestRaiseRequect < Test::Unit::TestCase
+  #
+  def setup
+    @validator = UTF8::Validator.new
+  end
+
+  #
+  def teardown
+    @validator = nil
+  end
+
+  # ISO-8859-1 characters
+  def test_0010_check_raise_iso
+    #
+    assert_raise(UTF8::ValidationError) do
+     !@validator.valid_encoding?(0x80.chr, true)
+    end
+  end
+
+  # A regular old bad byte
+  def test_0020_check_raise_badbyte
+    #
+    assert_raise(UTF8::ValidationError) do
+     !@validator.valid_encoding?("a" + 0xff.chr + "b", true)
+    end
+  end
+
+end
+

data/test/test_utf8_validator.rb

@@ -0,0 +1,226 @@
+# encoding: utf-8
+#
+require 'helper'
+
+#
+# == Purpose
+#
+# Tests for the #{UTF8::Validator} implementation.
+#
+class TestUtf8Validator < Test::Unit::TestCase
+  #
+  def setup
+    @validator = UTF8::Validator.new
+  end
+
+  #
+  def teardown
+    @validator = nil
+  end
+
+  #
+  def test_0010_linkages
+    assert_not_nil @validator
+  end
+
+  #--
+  # Validation should succeed for the following tests
+  #--
+
+  # Some simple ASCII characters
+  def test_0020_simple_ascii
+    test_data = [
+      "a",
+      "abcdefghijjlmnopqrstuvwxyz",
+      "\x00",
+    ]
+    test_data.each do |string|
+      assert @validator.valid_encoding?(string), "Simple ASCII: #{string}"
+      assert string.force_encoding("UTF-8").valid_encoding?, "Simple ASCII 19: #{string}" if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # All ASCII
+  def test_0030_all_ascii
+    0.upto(127) do |i|
+      assert @validator.valid_encoding?(i.chr), "All ASCII: #{i.chr}"
+      assert i.chr.force_encoding("UTF-8").valid_encoding?, "All ASCII 19: #{i.chr}" if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Simple UTF8 - 2 byte characters
+  def test_0040_simple_utf8_2byte
+    test_data = [
+      "\xc2\x80",
+      "\xc2\xbf",
+      "\xdf\x80",
+      "\xdf\xbf",
+    ]
+    test_data.each do |string|
+      assert @validator.valid_encoding?(string), "Simple UTF-8, 2bytes: #{string}"
+      assert string.force_encoding("UTF-8").valid_encoding?, "Simple UTF-8 19, 2bytes: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Simple UTF8 - 3 byte characters
+  def test_0050_simple_utf8_3byte
+    test_data = [
+      "\xe0\xa0\x80",
+      "\xe0\xbf\x80",
+      "\xe0\xa0\xbf",
+      "\xe0\xbf\xbf",
+    ]
+    test_data.each do |string|
+      assert @validator.valid_encoding?(string), "Simple UTF-8, 3bytes: #{string}"
+      assert string.force_encoding("UTF-8").valid_encoding?, "Simple UTF-8 19, 3bytes: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Simple UTF8 - 4 byte characters
+  def test_0060_simple_utf8_4byte
+    test_data = [
+      "\xf1\x80\x80\x80",
+      "\xf1\xbf\xbf\xbf",
+      "\xf2\x80\x80\x80",
+      "\xf2\xbf\xbf\xbf",
+      "\xf3\x80\x80\x80",
+      "\xf3\xbf\xbf\xbf",
+    ]
+    test_data.each do |string|
+      assert @validator.valid_encoding?(string), "Simple UTF-8, 4bytes: #{string}"
+      assert string.force_encoding("UTF-8").valid_encoding?, "Simple UTF-8 19, 4bytes: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  #--
+  # Validation should fail for the following tests
+  #--
+
+
+  # ISO-8859-1 (C1 points)
+  def test0510_iso_5559_1
+    0x80.upto(0x9f) do |i|
+      assert !@validator.valid_encoding?(i.chr), "ISO-8859-1: #{i}"
+      assert !i.chr.force_encoding("UTF-8").valid_encoding?, "ISO-8859-1 19: #{i}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # UTF-16 Surrogate Halves
+  def test0520_utf16_surrogate_halves
+    test_data = [
+      "\xed\xa0\x80", # u-800 (lowest)
+      "\xed\xbf\xbf", # u-fff (highest)
+    ]
+    test_data.each do |string|
+      assert !@validator.valid_encoding?(string), "UTF-16 Surrogate Halves: #{string}"
+      assert !string.force_encoding("UTF-8").valid_encoding?, "UTF-16 Surrogate Halves 19: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Invalid single bytes
+  def test0530_invalid_single_bytes
+    test_data = [
+      "\xc0",
+      "\xc1",
+      "\xf5","\xf6","\xf7","\xf8","\xf9","\xfa","\xfb","\xfc",
+      "\xfd","\xfe","\xff",
+    ]
+    test_data.each do |string|
+      assert !@validator.valid_encoding?(string), "Invalid single bytes: #{string}"
+      assert !string.force_encoding("UTF-8").valid_encoding?, "Invalid single bytes 10: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Not shortest representation
+  def test0540_not_shortest
+    test_data = [
+      "\xc0\x80",
+      "\xe0\x80\x80",
+      "\xf0\x80\x80\x80",
+      "\xf8\x80\x80\x80\x80",
+      "\xfc\x80\x80\x80\x80\x80",
+    ]
+    test_data.each do |string|
+      assert !@validator.valid_encoding?(string), "Not shortest: #{string}"
+      assert !string.force_encoding("UTF-8").valid_encoding?, "Not shortest 19: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Truncated last character
+  def test0550_truncated_last
+    test_data = [
+      "\xc2",  # truncated 2 byte characters
+      "\xdf",
+      "\xe0\xa0", # truncated 3 byte characters
+      "\xe0\xbf",
+      "\xf1\x80\x80", # truncated 4 byte characters
+      "\xf1\xbf\xbf",
+      "\xf2\x80\x80",
+      "\xf2\xbf\xbf",
+      "\xf3\x80\x80",
+      "\xf3\xbf\xbf",
+    ]
+    test_data.each do |string|
+      assert !@validator.valid_encoding?(string), "truncated last: #{string}"
+      assert !string.force_encoding("UTF-8").valid_encoding?, "truncated last 19: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Truncated in good text
+  def test0560_truncated_in_good
+    test_data = [
+      "\xc2",  # truncated 2 byte characters
+      "\xdf",
+      "\xe0\xa0", # truncated 3 byte characters
+      "\xe0\xbf",
+      "\xf1\x80\x80", # truncated 4 byte characters
+      "\xf1\xbf\xbf",
+      "\xf2\x80\x80",
+      "\xf2\xbf\xbf",
+      "\xf3\x80\x80",
+      "\xf3\xbf\xbf",
+    ]
+    test_data.each do |string|
+      string = "a" + string + "b"
+      assert !@validator.valid_encoding?(string), "truncated in good: #{string}"
+      assert !string.force_encoding("UTF-8").valid_encoding?, "truncated in good 19: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+  # Miscellaneous Bad
+  def test0570_miscellaneous_bad
+    # perhaps some duplication here
+    test_data = [
+      "bad byte: \372",
+      "\004\b{\f:\tbody\"\001\207\004\b{\b:\016statusmsg\"\aOK:\017statuscodei\000:\tdata{\t:\voutput\"3Enabled, not running, last run 693 seconds ago:\frunningi\000:\fenabledi\006:\flastrunl+\aE\021\022M:\rsenderid\"\032xx.xx.xx.xx:\016requestid\"%849d647bbe3e421ea19ac9f947bbdde4:\020senderagent\"\fpuppetd:\016msgtarget\"%/topic/mcollective.puppetd.reply:\thash\"\001\257ZdQqtaDmmdD0jZinnEcpN+YbkxQDn8uuCnwsQdvGHau6d+gxnnfPLUddWRSb\nZNMs+sQUXgJNfcV1eVBn1H+Z8QQmzYXVDMqz7J43jmgloz5PsLVbN9K3PmX/\ngszqV/WpvIyAqm98ennWqSzpwMuiCC4q2Jr3s3Gm6bUJ6UkKXnY=\n:\fmsgtimel+\a\372\023\022M",
+      "\207",
+      "\xf4\x90\x80\x80",
+      "\xbf",
+      "\xe0\x9f\xbf",
+      "\xf0\x8f\xbf\xbf",
+      "\xf8\x87\xbf\xbf\xbf",
+      "\xfc\x83\xbf\xbf\xbf\xbf",
+      "\xc0\x80",
+      "\xe0\x80\x80",
+      "\xf0\x80\x80\x80",
+      "\xf8\x80\x80\x80\x80",
+      "\xfc\x80\x80\x80\x80\x80",
+      "\xed\xa0\x80",
+      "\xed\xad\xbf",
+      "\xed\xae\x80",
+      "\xed\xaf\xbf",
+      "\xed\xb0\x80",
+      "\xed\xbe\x80",
+      "\xed\xbf\xbf",
+      "\xc0\x00", # too long for \x00
+      "\xe0\x00\x00", # too long for \x00
+      "\xf0\x00\x00\x00", # too long for \x00
+    ]
+    test_data.each do |string|
+      assert !@validator.valid_encoding?(string), "miscellaneous bad: #{string}"
+      assert !string.force_encoding("UTF-8").valid_encoding?, "miscellaneous bad 19: #{string}"  if RUBY_VERSION =~ /1\.9/
+    end
+  end
+
+end
+

data/utf8_validator.gemspec

@@ -0,0 +1,69 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{utf8_validator}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Guy Allard"]
+  s.date = %q{2011-01-25}
+  s.description = %q{A State Machine implementation of a UTF-8 Encoding 
+Validation algorithm.}
+  s.email = %q{allard.guy.m@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/utf8_validator.rb",
+    "lib/validation/errors.rb",
+    "lib/validation/validator.rb",
+    "test/helper.rb",
+    "test/test_raise_request.rb",
+    "test/test_utf8_validator.rb",
+    "utf8_validator.gemspec"
+  ]
+  s.homepage = %q{http://github.com/gmallard/utf8_validator}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{A UTF-8 Encoding Validator.}
+  s.test_files = [
+    "test/helper.rb",
+    "test/test_raise_request.rb",
+    "test/test_utf8_validator.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 2.1.2"])
+    else
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 2.1.2"])
+    end
+  else
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 2.1.2"])
+  end
+end
+

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification 
+name: utf8_validator
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Guy Allard
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-25 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  type: :development
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 1
+        - 2
+        version: 2.1.2
+  type: :development
+  prerelease: false
+  version_requirements: *id004
+description: |-
+  A State Machine implementation of a UTF-8 Encoding 
+  Validation algorithm.
+email: allard.guy.m@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/utf8_validator.rb
+- lib/validation/errors.rb
+- lib/validation/validator.rb
+- test/helper.rb
+- test/test_raise_request.rb
+- test/test_utf8_validator.rb
+- utf8_validator.gemspec
+has_rdoc: true
+homepage: http://github.com/gmallard/utf8_validator
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: -1120544117494340473
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A UTF-8 Encoding Validator.
+test_files: 
+- test/helper.rb
+- test/test_raise_request.rb
+- test/test_utf8_validator.rb