gigo 2.0.0 → 2.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 712fefdcd90415f02b05a5c1d71b59c7650aeed7
+  data.tar.gz: 8645a95acd55340917bad0d9fc080f8325c26fef
+SHA512:
+  metadata.gz: 47388733658210b84307e259798c395ae97d07524a19097853605da9071306e0ae0eacb5f8db91b693e22fa16126615af10a3a88923df80c7a17d7ee6fffc5ae
+  data.tar.gz: 2c1b9b8ef11693facef2e5a4ddd0f42705a4cb8629b182949ccce9325f95582e45a7c8b1c18843477935841319816df2c4711b8a1556dbbef97edc0956b9c369

data/Appraisals

@@ -1,3 +1,7 @@
+appraise 'activesupport23' do
+  gem 'activesupport', '~> 2.3.0'
+  gem 'iconv'
+end
 
 appraise 'activesupport30' do
   gem 'activesupport', '~> 3.0.0'
@@ -12,6 +16,9 @@ appraise 'activesupport32' do
 end
 
 appraise 'activesupport40' do
-  gem 'activesupport', :github => 'rails/rails'
+  gem 'activesupport', '~> 4.0.0'
 end
 
+appraise 'activesupport41' do
+  gem 'activesupport', '~> 4.1.0'
+end

data/README.md

@@ -16,11 +16,11 @@ Simple, just pass a string to `GIGO.load`. Nil values or properly encoded string
 GIGO.load "€20 – “Woohoo”"
 ```
 
-Lets say you have a `comments` column on an ActiveRecord model which is not guaranteed to come back per your default external encoding.
+Lets say you have a `comments` column on an ActiveRecord model which is not guaranteed to come back per your default external encoding. However, take a look at the [gigo-activerecord](https://github.com/customink/gigo-activerecord) gem.
 
 ```ruby
 def comments
-  GIGO.load read_attribute(:comments)
+  GIGO.load(super)
 end
 ```
 
@@ -36,10 +36,18 @@ GIGO transcoders can be any module or class that implements the `transcode` meth
 
 GIGO attempts to use each in that order. Upon successful transcoding, we use the [EnsureValidEncoding](http://github.com/jrochkind/ensure_valid_encoding) gem to force an encoding to match the `GIGO.encoding` while removing any non-convertable characters.
 
+#### Rails/ActiveSupport v2.3.x
+
+When using GIGO with Rails/ActiveSupport 2.3., the `GIGO::Transcoders::CharlockHolmes` transcoder will be inserted before the `GIGO::Transcoders::ActiveSupport` one. This is needed because ActiveSupport's multibyte char support is weak in version 2.3.x. You will need to add this to your applications `Gemfile` since `CharlockHolmes` requires it.
+
+```ruby
+gem 'iconv'
+```
+
 
 ## Contributing
 
-GIGO is fully tested with ActiveSupport 3.0 to 4 and upward. If you detect a problem, open up a github issue or fork the repo and help out. After you fork or clone the repository, the following commands will get you up and running on the test suite. 
+GIGO is fully tested with ActiveSupport 3.0 to 4 and upward. If you detect a problem, open up a github issue or fork the repo and help out. After you fork or clone the repository, the following commands will get you up and running on the test suite.
 
 ```shell
 $ bundle install

data/gemfiles/activesupport23.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 2.3.0"
+gem "iconv"
+
+gemspec :path=>"../"

data/gemfiles/activesupport23.gemfile.lock

@@ -0,0 +1,35 @@
+PATH
+  remote: ../
+  specs:
+    gigo (2.0.0)
+      activesupport (>= 2.3)
+      charlock_holmes (~> 0.7)
+      ensure_valid_encoding (~> 0.5.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (2.3.18)
+    appraisal (1.0.2)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    charlock_holmes (0.7.3)
+    ensure_valid_encoding (0.5.3)
+    i18n (0.6.11)
+    iconv (1.0.4)
+    minitest (5.4.2)
+    rake (10.3.2)
+    thor (0.19.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 2.3.0)
+  appraisal
+  gigo!
+  i18n
+  iconv
+  minitest
+  rake

data/gemfiles/activesupport30.gemfile.lock

@@ -1,21 +1,25 @@
 PATH
-  remote: /Users/kencollins/Repositories/customink/gigo
+  remote: ../
   specs:
     gigo (2.0.0)
-      activesupport (>= 3.0)
+      activesupport (>= 2.3)
+      charlock_holmes (~> 0.7)
       ensure_valid_encoding (~> 0.5.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
     activesupport (3.0.20)
-    appraisal (0.5.2)
+    appraisal (1.0.2)
       bundler
       rake
+      thor (>= 0.14.0)
+    charlock_holmes (0.7.3)
     ensure_valid_encoding (0.5.3)
-    i18n (0.6.4)
-    minitest (5.0.1)
-    rake (10.0.4)
+    i18n (0.6.11)
+    minitest (5.4.2)
+    rake (10.3.2)
+    thor (0.19.1)
 
 PLATFORMS
   ruby
@@ -25,5 +29,5 @@ DEPENDENCIES
   appraisal
   gigo!
   i18n
-  minitest (~> 5.0)
+  minitest
   rake

data/gemfiles/activesupport31.gemfile.lock

@@ -1,23 +1,27 @@
 PATH
-  remote: /Users/kencollins/Repositories/customink/gigo
+  remote: ../
   specs:
     gigo (2.0.0)
-      activesupport (>= 3.0)
+      activesupport (>= 2.3)
+      charlock_holmes (~> 0.7)
       ensure_valid_encoding (~> 0.5.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (3.1.10)
-      multi_json (>= 1.0, < 1.3)
-    appraisal (0.5.2)
+    activesupport (3.1.12)
+      multi_json (~> 1.0)
+    appraisal (1.0.2)
       bundler
       rake
+      thor (>= 0.14.0)
+    charlock_holmes (0.7.3)
     ensure_valid_encoding (0.5.3)
-    i18n (0.6.4)
-    minitest (5.0.1)
-    multi_json (1.2.0)
-    rake (10.0.4)
+    i18n (0.6.11)
+    minitest (5.4.2)
+    multi_json (1.10.1)
+    rake (10.3.2)
+    thor (0.19.1)
 
 PLATFORMS
   ruby
@@ -27,5 +31,5 @@ DEPENDENCIES
   appraisal
   gigo!
   i18n
-  minitest (~> 5.0)
+  minitest
   rake

data/gemfiles/activesupport32.gemfile.lock

@@ -1,24 +1,28 @@
 PATH
-  remote: /Users/kencollins/Repositories/customink/gigo
+  remote: ../
   specs:
     gigo (2.0.0)
-      activesupport (>= 3.0)
+      activesupport (>= 2.3)
+      charlock_holmes (~> 0.7)
       ensure_valid_encoding (~> 0.5.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (3.2.12)
-      i18n (~> 0.6)
+    activesupport (3.2.19)
+      i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
-    appraisal (0.5.2)
+    appraisal (1.0.2)
       bundler
       rake
+      thor (>= 0.14.0)
+    charlock_holmes (0.7.3)
     ensure_valid_encoding (0.5.3)
-    i18n (0.6.4)
-    minitest (5.0.1)
-    multi_json (1.7.3)
-    rake (10.0.4)
+    i18n (0.6.11)
+    minitest (5.4.2)
+    multi_json (1.10.1)
+    rake (10.3.2)
+    thor (0.19.1)
 
 PLATFORMS
   ruby
@@ -28,5 +32,5 @@ DEPENDENCIES
   appraisal
   gigo!
   i18n
-  minitest (~> 5.0)
+  minitest
   rake

data/gemfiles/activesupport40.gemfile

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", :github=>"rails/rails"
+gem "activesupport", "~> 4.0.0"
 
 gemspec :path=>"../"

data/gemfiles/activesupport40.gemfile.lock

@@ -1,44 +1,41 @@
-GIT
-  remote: git://github.com/rails/rails.git
-  revision: d71b0935a956ebedea0bb975525fc0f5672e4088
-  specs:
-    activesupport (4.1.0.beta)
-      i18n (~> 0.6, >= 0.6.4)
-      json (~> 1.7)
-      minitest (~> 5.0)
-      thread_safe (~> 0.1)
-      tzinfo (~> 0.3.37)
-
 PATH
-  remote: /Users/kencollins/Repositories/customink/gigo
+  remote: ../
   specs:
     gigo (2.0.0)
-      activesupport (>= 3.0)
+      activesupport (>= 2.3)
+      charlock_holmes (~> 0.7)
       ensure_valid_encoding (~> 0.5.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    appraisal (0.5.2)
+    activesupport (4.0.11)
+      i18n (~> 0.6, >= 0.6.9)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    appraisal (1.0.2)
       bundler
       rake
-    atomic (1.1.9)
+      thor (>= 0.14.0)
+    charlock_holmes (0.7.3)
     ensure_valid_encoding (0.5.3)
-    i18n (0.6.4)
-    json (1.8.0)
-    minitest (5.0.1)
-    rake (10.0.4)
-    thread_safe (0.1.0)
-      atomic
-    tzinfo (0.3.37)
+    i18n (0.6.11)
+    minitest (4.7.5)
+    multi_json (1.10.1)
+    rake (10.3.2)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    tzinfo (0.3.42)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport!
+  activesupport (~> 4.0.0)
   appraisal
   gigo!
   i18n
-  minitest (~> 5.0)
+  minitest
   rake

data/gemfiles/activesupport41.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 4.1.0"
+
+gemspec :path=>"../"

data/gemfiles/activesupport41.gemfile.lock

@@ -0,0 +1,42 @@
+PATH
+  remote: ../
+  specs:
+    gigo (2.0.0)
+      activesupport (>= 2.3)
+      charlock_holmes (~> 0.7)
+      ensure_valid_encoding (~> 0.5.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    appraisal (1.0.2)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    charlock_holmes (0.7.3)
+    ensure_valid_encoding (0.5.3)
+    i18n (0.6.11)
+    json (1.8.1)
+    minitest (5.4.2)
+    rake (10.3.2)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 4.1.0)
+  appraisal
+  gigo!
+  i18n
+  minitest
+  rake

data/gigo.gemspec

@@ -15,10 +15,11 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency     'activesupport', '>= 3.0'
+  gem.add_runtime_dependency     'activesupport', '>= 2.3'
   gem.add_runtime_dependency     'ensure_valid_encoding', '~> 0.5.3'
+  gem.add_runtime_dependency     'charlock_holmes', '~> 0.7'
   gem.add_development_dependency 'appraisal'
   gem.add_development_dependency 'i18n' # Older ActiveSupport does not have a proper dep.
   gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'minitest', '~> 5.0'
+  gem.add_development_dependency 'minitest'
 end

data/lib/gigo.rb

@@ -1,6 +1,8 @@
 require 'active_support/all'
+require 'active_support/version'
 require 'ensure_valid_encoding'
 require 'gigo/transcoders'
+require 'gigo/transcoders/charlock_holmes' if ActiveSupport::VERSION::STRING < '3.0'
 require 'gigo/transcoders/active_support'
 require 'gigo/transcoders/blind'
 require 'gigo/version'
@@ -8,7 +10,6 @@ require 'gigo/version'
 module GIGO
 
   mattr_accessor :encoding
-  self.encoding = Encoding.default_internal || Encoding::UTF_8
 
   def self.load(data, options = {})
     return data if data.nil? || !data.acts_like?(:string)
@@ -18,6 +19,11 @@ module GIGO
     encoded_string
   end
 
+  def self.reset_encoding!
+    self.encoding = Encoding.default_internal || Encoding::UTF_8
+  end
+  reset_encoding!
+
 
   protected
 

data/lib/gigo/transcoders.rb

@@ -1,6 +1,6 @@
 module GIGO
   module Transcoders
-  
+
   end
   mattr_accessor :transcoders
   self.transcoders = []

data/lib/gigo/transcoders/active_support.rb

@@ -7,7 +7,7 @@ module GIGO
       def self.transcode(data)
         ::ActiveSupport::Multibyte.proxy_class.new(data).tidy_bytes.to_s
       end
-      
+
     end
   end
 end

data/lib/gigo/transcoders/blind.rb

@@ -7,7 +7,7 @@ module GIGO
       def self.transcode(data)
         data.encode GIGO.encoding, :undef => :replace, :invalid => :replace
       end
-      
+
     end
   end
 end

data/lib/gigo/transcoders/charlock_holmes.rb

@@ -0,0 +1,26 @@
+require 'iconv'
+require 'charlock_holmes'
+
+module GIGO
+  module Transcoders
+    module CharlockHolmes
+
+      GIGO.transcoders << self
+
+      def self.transcode(data)
+        ::CharlockHolmes::Converter.convert data, detect_encoding(data), GIGO.encoding.name
+      end
+
+      private
+
+      def self.detect_encoding(data)
+        encoding = ::CharlockHolmes::EncodingDetector.detect(data)[:encoding]
+        case encoding
+        when 'UTF-16BE' then 'CP1252'
+        else encoding
+        end
+      end
+
+    end
+  end
+end

data/lib/gigo/version.rb

@@ -1,3 +1,3 @@
 module GIGO
-  VERSION = "2.0.0"
+  VERSION = "2.1.0"
 end

data/test/cases/gigo_test.rb

@@ -30,7 +30,7 @@ module GIGO
     end
 
     describe '.load' do
-      
+
       it 'ignores if string is not present' do
         GIGO.load('').must_equal ''
         GIGO.load(nil).must_be_nil
@@ -59,7 +59,7 @@ module GIGO
         GIGO.load(data_medico_unknown).must_equal "Med�co"
       end
 
-      it 'makes sure UTF-8 data read in as US-ASCII us fixed' do
+      it 'makes sure UTF-8 data read in as US-ASCII is fixed' do
         GIGO.load(data_medico_utf8.force_encoding('US-ASCII')).must_equal 'Medíco'
       end
 
@@ -79,7 +79,7 @@ module GIGO
         GIGO.load(data_medico_iso88591.force_encoding('US-ASCII')).must_equal 'Medíco'
       end
 
-      it 'can make sure to it is really a valid encoding afterward' do
+      it 'readly bad data can be html escaped afterward' do
         html_escape GIGO.load(data_really_bad)
       end
 
@@ -88,13 +88,14 @@ module GIGO
     describe '.transcoders' do
 
       it 'is an array of default transcoders' do
-        GIGO.transcoders.must_equal [
-          GIGO::Transcoders::ActiveSupport,
-          GIGO::Transcoders::Blind
-        ]
+        if ActiveSupport::VERSION::STRING < '3.0'
+          GIGO.transcoders.must_equal [GIGO::Transcoders::CharlockHolmes, GIGO::Transcoders::ActiveSupport, GIGO::Transcoders::Blind]
+        else
+          GIGO.transcoders.must_equal [GIGO::Transcoders::ActiveSupport, GIGO::Transcoders::Blind]
+        end
       end
 
     end
-    
+
   end
 end

data/test/test_helper.rb

@@ -7,20 +7,18 @@ module GIGO
   class TestCase < MiniTest::Spec
 
     include ERB::Util
+    @@default_internal_encoding = Encoding.default_internal
 
-    before { setup_gigo }
-    after  { teardown_gigo }
+    before { reset_encodings }
 
 
     private
 
-    def setup_gigo
-      @_default_gigo_encoding = GIGO.encoding
+    def reset_encodings
+      Encoding.default_internal = @@default_internal_encoding
+      GIGO.reset_encoding!
     end
 
-    def teardown_gigo
-      GIGO.encoding = @_default_gigo_encoding
-    end
-    
+
   end
 end

metadata

@@ -1,112 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: gigo
 version: !ruby/object:Gem::Version
-  version: 2.0.0
-  prerelease: 
+  version: 2.1.0
 platform: ruby
 authors:
 - Ken Collins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-19 00:00:00.000000000 Z
+date: 2014-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: ensure_valid_encoding
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.3
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: i18n
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
 description: Garbage in, garbage out. Fix ruby encoded strings at all costs.
 email:
 - kcollins@customink.com
@@ -114,12 +115,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Appraisals
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- gemfiles/activesupport23.gemfile
+- gemfiles/activesupport23.gemfile.lock
 - gemfiles/activesupport30.gemfile
 - gemfiles/activesupport30.gemfile.lock
 - gemfiles/activesupport31.gemfile
@@ -128,43 +131,39 @@ files:
 - gemfiles/activesupport32.gemfile.lock
 - gemfiles/activesupport40.gemfile
 - gemfiles/activesupport40.gemfile.lock
+- gemfiles/activesupport41.gemfile
+- gemfiles/activesupport41.gemfile.lock
 - gigo.gemspec
 - lib/gigo.rb
 - lib/gigo/transcoders.rb
 - lib/gigo/transcoders/active_support.rb
 - lib/gigo/transcoders/blind.rb
+- lib/gigo/transcoders/charlock_holmes.rb
 - lib/gigo/version.rb
 - test/cases/gigo_test.rb
 - test/test_helper.rb
 homepage: http://github.com/customink/gigo
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 952284011809381427
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 952284011809381427
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.4.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: The gigo gem aims to solve bad data, likely from a legacy database. It is
   an anti-pattern and you should really consider standardizing what encoding you both
   put in and take out of your data stores.