gigo 1.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Appraisals

@@ -0,0 +1,17 @@
+
+appraise 'activesupport30' do
+  gem 'activesupport', '~> 3.0.0'
+end
+
+appraise 'activesupport31' do
+  gem 'activesupport', '~> 3.1.0'
+end
+
+appraise 'activesupport32' do
+  gem 'activesupport', '~> 3.2.0'
+end
+
+appraise 'activesupport40' do
+  gem 'activesupport', :github => 'rails/rails'
+end
+

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Ken Collins
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+
+# GIGO (Garbage In, Garbage Out)
+
+Or better yet, Garbage In, Gold Out! - The GIGO gem aims to fix ruby string encodings at all costs!
+
+The GIGO gem is not likely the proper solutions. If you have bad encodings in your database, you should fix them and write consistent encodings. That said, if you have no other choice, GIGO can help.
+
+This gem depends on one of the many public forks of `CharDet` for ruby. Since `CharDet` is not a public gem and following proper semantic versioning, we have decided to vendor the [kirillrdy/rchardet](http://github.com/kirillrdy/rchardet) repo. We have even made sure that our vendored version stays in our namesacpe by using `GIGO::CharDet`. So if you have another version bundled, feel confident that the two will not conflict.
+
+We use `GIGO::CharDet` to do the grunt work of finding the proper encoding of an untrusted string. Once found, we use the [EnsureValidEncoding](http://github.com/jrochkind/ensure_valid_encoding) gem to either force an encoding while removing any non-convertable characters.
+
+
+## Usage
+
+Simple, just pass a string to `GIGO.load`. Nil values or properly encoded strings are returned. Else, `GIGO` will do its best to convert and force your default internal (or UTF-8) encoding.
+
+```ruby
+  GIGO.load "€20 – “Woohoo”"
+```
+
+Lets say you have a `comments` column on an ActiveRecord model which is not guaranteed to come back per your default external encoding.
+
+```ruby
+def comments
+  GIGO.load read_attribute(:comments)
+end
+```
+
+
+## Toe Dough List
+
+Remvoe CharDet and look at something like [CharlockHolmes](https://github.com/brianmario/charlock_holmes). I had install problems with this and it also failed a few initial tire kicks. See [my notes](https://gist.github.com/metaskills/5029604) here on the topic.
+
+
+## Contributing
+
+GIGO is fully tested with ActiveSupport 3.0 to 4 and upward. If you detect a problem, open up a github issue or fork the repo and help out. After you fork or clone the repository, the following commands will get you up and running on the test suite. 
+
+```shell
+$ bundle install
+$ bundle exec rake appraisal:setup
+$ bundle exec rake appraisal test
+```
+
+We use the [appraisal](https://github.com/thoughtbot/appraisal) gem from Thoughtbot to help us generate the individual gemfiles for each ActiveSupport version and to run the tests locally against each generated Gemfile. The `rake appraisal test` command actually runs our test suite against all Rails versions in our `Appraisal` file. If you want to run the tests for a specific Rails version, use `rake -T` for a list. For example, the following command will run the tests for Rails 3.2 only.
+
+```shell
+$ bundle exec rake appraisal:rails32 test
+```
+

data/Rakefile

@@ -0,0 +1,18 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'appraisal'
+
+Rake::TestTask.new   do |t|
+  t.libs = ['lib','test']
+  t.test_files = Dir.glob('test/**/*_test.rb').sort
+  t.verbose = true
+end
+
+task :default => :test
+
+desc "Setup Appraisal."
+task 'appraisal:setup' do
+  Rake::Task['appraisal:cleanup'].invoke
+  Rake::Task['appraisal:gemfiles'].invoke
+  Rake::Task['appraisal:install'].invoke
+end

data/gemfiles/activesupport30.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.0.0"
+
+gemspec :path=>"../"

data/gemfiles/activesupport30.gemfile.lock

@@ -0,0 +1,29 @@
+PATH
+  remote: /Users/kencollins/Repositories/gigo
+  specs:
+    gigo (1.0.0)
+      activesupport (>= 3.0)
+      ensure_valid_encoding (~> 0.5.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (3.0.20)
+    appraisal (0.5.1)
+      bundler
+      rake
+    ensure_valid_encoding (0.5.3)
+    minitest (4.6.1)
+    minitest-emoji (1.0.0)
+    rake (10.0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 3.0.0)
+  appraisal
+  gigo!
+  minitest
+  minitest-emoji
+  rake

data/gemfiles/activesupport31.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.1.0"
+
+gemspec :path=>"../"

data/gemfiles/activesupport31.gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: /Users/kencollins/Repositories/gigo
+  specs:
+    gigo (1.0.0)
+      activesupport (>= 3.0)
+      ensure_valid_encoding (~> 0.5.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (3.1.10)
+      multi_json (>= 1.0, < 1.3)
+    appraisal (0.5.1)
+      bundler
+      rake
+    ensure_valid_encoding (0.5.3)
+    minitest (4.6.1)
+    minitest-emoji (1.0.0)
+    multi_json (1.2.0)
+    rake (10.0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 3.1.0)
+  appraisal
+  gigo!
+  minitest
+  minitest-emoji
+  rake

data/gemfiles/activesupport32.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.2.0"
+
+gemspec :path=>"../"

data/gemfiles/activesupport32.gemfile.lock

@@ -0,0 +1,33 @@
+PATH
+  remote: /Users/kencollins/Repositories/gigo
+  specs:
+    gigo (1.0.0)
+      activesupport (>= 3.0)
+      ensure_valid_encoding (~> 0.5.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (3.2.12)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    appraisal (0.5.1)
+      bundler
+      rake
+    ensure_valid_encoding (0.5.3)
+    i18n (0.6.1)
+    minitest (4.6.1)
+    minitest-emoji (1.0.0)
+    multi_json (1.6.1)
+    rake (10.0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 3.2.0)
+  appraisal
+  gigo!
+  minitest
+  minitest-emoji
+  rake

data/gemfiles/activesupport40.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", :github=>"rails/rails"
+
+gemspec :path=>"../"

data/gemfiles/activesupport40.gemfile.lock

@@ -0,0 +1,45 @@
+GIT
+  remote: git://github.com/rails/rails.git
+  revision: 23048551fdb679e694d8245e9f8c969ed7a71f40
+  specs:
+    activesupport (4.0.0.beta)
+      i18n (~> 0.6)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.33)
+
+PATH
+  remote: /Users/kencollins/Repositories/gigo
+  specs:
+    gigo (1.0.0)
+      activesupport (>= 3.0)
+      ensure_valid_encoding (~> 0.5.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (0.5.1)
+      bundler
+      rake
+    atomic (1.0.1)
+    ensure_valid_encoding (0.5.3)
+    i18n (0.6.1)
+    minitest (4.6.1)
+    minitest-emoji (1.0.0)
+    multi_json (1.6.1)
+    rake (10.0.3)
+    thread_safe (0.1.0)
+      atomic
+    tzinfo (0.3.35)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport!
+  appraisal
+  gigo!
+  minitest
+  minitest-emoji
+  rake

data/gigo.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gigo/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'gigo'
+  gem.version       = GIGO::VERSION
+  gem.authors       = ["Ken Collins"]
+  gem.email         = ["kcollins@customink.com"]
+  gem.description   = 'Garbage in, garbage out. Fix ruby encoded strings at all costs.'
+  gem.summary       = 'The gigo gem aims to solve bad data, likely from a legacy database. It is an anti-pattern and you should really consider standardizing what encoding you both put in and take out of your data stores.'
+  gem.homepage      = 'http://github.com/customink/gigo'
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_runtime_dependency     'activesupport', '>= 3.0'
+  gem.add_runtime_dependency     'ensure_valid_encoding', '~> 0.5.3'
+  gem.add_development_dependency 'appraisal'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'minitest'
+  gem.add_development_dependency 'minitest-emoji'
+end

data/lib/gigo.rb

@@ -0,0 +1,34 @@
+require 'active_support/core_ext/object/acts_like'
+require 'active_support/core_ext/string/behavior'
+require 'ensure_valid_encoding'
+require 'gigo/rchardet'
+require 'gigo/version'
+
+module GIGO
+
+  def self.load(data)
+    return data if data.nil? || !data.acts_like?(:string)
+    encoded_string = safe_detect_and_encoder(data)
+    return data if data.encoding == forced_encoding && data == encoded_string
+    encoded_string
+  end
+
+
+  protected
+
+  def self.safe_detect_and_encoder(data)
+    string = data
+    begin
+      encoding = CharDet.detect(string.dup)['encoding'] || string.encoding || Encoding.default_internal || forced_encoding
+      string = string.force_encoding(encoding).encode forced_encoding, :undef => :replace, :invalid => :replace
+    rescue Exception => e
+      string = string.encode forced_encoding, :undef => :replace, :invalid => :replace
+    end
+    EnsureValidEncoding.ensure_valid_encoding string, invalid: :replace, replace: "?"
+  end
+
+  def self.forced_encoding
+    Encoding.default_internal || Encoding::UTF_8
+  end
+  
+end

data/lib/gigo/rchardet.rb

@@ -0,0 +1,67 @@
+######################## BEGIN LICENSE BLOCK ########################
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'gigo/rchardet/charsetprober'
+require 'gigo/rchardet/mbcharsetprober'
+
+require 'gigo/rchardet/big5freq'
+require 'gigo/rchardet/big5prober'
+require 'gigo/rchardet/chardistribution'
+require 'gigo/rchardet/charsetgroupprober'
+
+require 'gigo/rchardet/codingstatemachine'
+require 'gigo/rchardet/constants'
+require 'gigo/rchardet/escprober'
+require 'gigo/rchardet/escsm'
+require 'gigo/rchardet/eucjpprober'
+require 'gigo/rchardet/euckrfreq'
+require 'gigo/rchardet/euckrprober'
+require 'gigo/rchardet/euctwfreq'
+require 'gigo/rchardet/euctwprober'
+require 'gigo/rchardet/gb2312freq'
+require 'gigo/rchardet/gb2312prober'
+require 'gigo/rchardet/hebrewprober'
+require 'gigo/rchardet/jisfreq'
+require 'gigo/rchardet/jpcntx'
+require 'gigo/rchardet/langbulgarianmodel'
+require 'gigo/rchardet/langcyrillicmodel'
+require 'gigo/rchardet/langgreekmodel'
+require 'gigo/rchardet/langhebrewmodel'
+require 'gigo/rchardet/langhungarianmodel'
+require 'gigo/rchardet/langthaimodel'
+require 'gigo/rchardet/latin1prober'
+
+require 'gigo/rchardet/mbcsgroupprober'
+require 'gigo/rchardet/mbcssm'
+require 'gigo/rchardet/sbcharsetprober'
+require 'gigo/rchardet/sbcsgroupprober'
+require 'gigo/rchardet/sjisprober'
+require 'gigo/rchardet/universaldetector'
+require 'gigo/rchardet/utf8prober'
+
+module GIGO
+module CharDet
+  VERSION = "1.3"
+  def CharDet.detect(aBuf)
+    u = UniversalDetector.new
+    u.reset
+    u.feed(aBuf)
+    u.close
+    u.result
+  end
+end
+end