gigo 1.0.0 → 1.1.0

data/gemfiles/activesupport30.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
-  remote: /Users/kencollins/Repositories/gigo
+  remote: /Users/kencollins/Repositories/customink/gigo
   specs:
-    gigo (1.0.0)
+    gigo (1.1.0)
       activesupport (>= 3.0)
       ensure_valid_encoding (~> 0.5.3)
 
@@ -13,9 +13,9 @@ GEM
       bundler
       rake
     ensure_valid_encoding (0.5.3)
-    minitest (4.6.1)
+    minitest (4.7.0)
     minitest-emoji (1.0.0)
-    rake (10.0.3)
+    rake (10.0.4)
 
 PLATFORMS
   ruby

data/gemfiles/activesupport31.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
-  remote: /Users/kencollins/Repositories/gigo
+  remote: /Users/kencollins/Repositories/customink/gigo
   specs:
-    gigo (1.0.0)
+    gigo (1.1.0)
       activesupport (>= 3.0)
       ensure_valid_encoding (~> 0.5.3)
 
@@ -14,10 +14,10 @@ GEM
       bundler
       rake
     ensure_valid_encoding (0.5.3)
-    minitest (4.6.1)
+    minitest (4.7.0)
     minitest-emoji (1.0.0)
     multi_json (1.2.0)
-    rake (10.0.3)
+    rake (10.0.4)
 
 PLATFORMS
   ruby

data/gemfiles/activesupport32.gemfile.lock

@@ -1,25 +1,25 @@
 PATH
-  remote: /Users/kencollins/Repositories/gigo
+  remote: /Users/kencollins/Repositories/customink/gigo
   specs:
-    gigo (1.0.0)
+    gigo (1.1.0)
       activesupport (>= 3.0)
       ensure_valid_encoding (~> 0.5.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (3.2.12)
-      i18n (~> 0.6)
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
       multi_json (~> 1.0)
     appraisal (0.5.1)
       bundler
       rake
     ensure_valid_encoding (0.5.3)
     i18n (0.6.1)
-    minitest (4.6.1)
+    minitest (4.7.0)
     minitest-emoji (1.0.0)
-    multi_json (1.6.1)
-    rake (10.0.3)
+    multi_json (1.7.2)
+    rake (10.0.4)
 
 PLATFORMS
   ruby

data/gemfiles/activesupport40.gemfile.lock

@@ -1,18 +1,18 @@
 GIT
   remote: git://github.com/rails/rails.git
-  revision: 23048551fdb679e694d8245e9f8c969ed7a71f40
+  revision: b6e5971e53746a79abc92ce9acea85ca14fab0b3
   specs:
-    activesupport (4.0.0.beta)
-      i18n (~> 0.6)
+    activesupport (4.0.0.beta1)
+      i18n (~> 0.6, >= 0.6.4)
       minitest (~> 4.2)
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
-      tzinfo (~> 0.3.33)
+      tzinfo (~> 0.3.37)
 
 PATH
-  remote: /Users/kencollins/Repositories/gigo
+  remote: /Users/kencollins/Repositories/customink/gigo
   specs:
-    gigo (1.0.0)
+    gigo (1.1.0)
       activesupport (>= 3.0)
       ensure_valid_encoding (~> 0.5.3)
 
@@ -24,14 +24,14 @@ GEM
       rake
     atomic (1.0.1)
     ensure_valid_encoding (0.5.3)
-    i18n (0.6.1)
-    minitest (4.6.1)
+    i18n (0.6.4)
+    minitest (4.7.0)
     minitest-emoji (1.0.0)
-    multi_json (1.6.1)
-    rake (10.0.3)
+    multi_json (1.7.2)
+    rake (10.0.4)
     thread_safe (0.1.0)
       atomic
-    tzinfo (0.3.35)
+    tzinfo (0.3.37)
 
 PLATFORMS
   ruby

data/lib/gigo.rb

@@ -1,3 +1,4 @@
+require 'active_support/multibyte'
 require 'active_support/core_ext/object/acts_like'
 require 'active_support/core_ext/string/behavior'
 require 'ensure_valid_encoding'
@@ -19,10 +20,14 @@ module GIGO
   def self.safe_detect_and_encoder(data)
     string = data
     begin
-      encoding = CharDet.detect(string.dup)['encoding'] || string.encoding || Encoding.default_internal || forced_encoding
-      string = string.force_encoding(encoding).encode forced_encoding, :undef => :replace, :invalid => :replace
+      string = ActiveSupport::Multibyte.proxy_class.new(string).tidy_bytes
     rescue Exception => e
-      string = string.encode forced_encoding, :undef => :replace, :invalid => :replace
+      begin
+        encoding = CharDet.detect(string.dup)['encoding'] || string.encoding || Encoding.default_internal || forced_encoding
+        string = string.force_encoding(encoding).encode forced_encoding, :undef => :replace, :invalid => :replace
+      rescue Exception => e
+        string = string.encode forced_encoding, :undef => :replace, :invalid => :replace
+      end
     end
     EnsureValidEncoding.ensure_valid_encoding string, invalid: :replace, replace: "?"
   end

data/lib/gigo/version.rb

@@ -1,3 +1,3 @@
 module GIGO
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/test/cases/gigo_test.rb

@@ -10,6 +10,7 @@ module GIGO
     let(:data_utf8)         { "€20 – “Woohoo”" }
     let(:data_bad_readin)   { "�20 � �Woohoo�" }
     let(:data_cp1252)       { data_utf8.encode('CP1252') }
+    let(:data_bin_apos)     { "won\x92t".force_encoding('binary') }
     let(:data_really_bad)   { "ed.Ã\u0083Ã\u0083\xC3" }
 
 
@@ -22,6 +23,10 @@ module GIGO
         GIGO.load(o).must_equal o
       end
 
+      it 'fixes windows apostrophe' do
+        GIGO.load(data_bin_apos).must_equal "won’t"
+      end
+
       it 'should allows properly encoded and marked strings to be passed thru' do
         GIGO.load(data_utf8).must_equal data_utf8
         GIGO.load(data_utf8_emoji).must_equal data_utf8_emoji
@@ -32,7 +37,7 @@ module GIGO
       end
 
       it 'allows really bad data to be encoded using default replace and question marks' do
-        GIGO.load(data_utf8_emoji.force_encoding('ASCII-8BIT')).must_equal '����'
+        GIGO.load(data_utf8_emoji.force_encoding('ASCII-8BIT')).must_equal data_utf8_emoji
       end
 
       it 'converts windows codepages that are poorly marked as another encoding' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gigo
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -183,7 +183,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1651701888858574901
+      hash: 2152698803205983593
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -192,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1651701888858574901
+      hash: 2152698803205983593
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25