ensure_valid_encoding 0.5.0 → 0.5.1

data/README.md

@@ -63,8 +63,8 @@ fixed = ensure_valid_encoding(bad_str)
 ## Rationale
 
 You are taking textual input from some external source. Could be user input, 
-could be a user-uploaded file of some kind, could be a a third party API, could
-be anything. 
+could be a user-uploaded file of some kind, could be anchient usenet archives,
+could be a a third party API response or web scrape, could be anything. 
 
 You know what character encoding the textual data _claims_ to be, what it
 _should_ be, and what it _usually_ is.  But occasionally it may have bad bytes
@@ -99,11 +99,10 @@ surprisingly tricky to do with the ruby 1.9.3 stdlib, or even with 'iconv'.
 So there you go, now you can do it with this gem. Maybe not as performant 
 as if it were implemented in C like stdlib char encoding routines, but, hey. 
 
-**Note well:** A buncha [people on reddit](http://www.reddit.com/r/ruby/comments/sfceq) don't see why you'd ever want to do
-this. If you agree, then, well, don't do it. 
-
-**Also:** I have [filed a feature request](https://bugs.ruby-lang.org/issues/6321) for ruby stdlib. Developer from 
-ruby core team also thinks it's an unusually odd thing to do. shrug. 
+ 
+**Note:** I have [filed a feature request](https://bugs.ruby-lang.org/issues/6321) for ruby stdlib. Developer from 
+ruby core team also thinks it's an unusually odd thing to do. Most of rubydom seems to agree. Ce la vie. Me and
+many of my colleagues need to do this all the time, but if you don't, don't do it. 
 
 ## Developing/Contributing
 

data/lib/ensure_valid_encoding.rb

@@ -29,7 +29,7 @@ module EnsureValidEncoding
   #     ensure_valid_encoding( some_string, :invalid => :replace)
   #     ensure_valid_encoding( some_string, :invalid => :replace, :replace => '')
   #     ensure_valid_encoding( some_string, :invalid => :replace, :replace => "*")
-  def self.ensure_valid_encoding(str, options = {})
+    def self.ensure_valid_encoding(str, options = {})
       # Can do nothing in ruby 1.8.x
       return str unless str.respond_to?(:encoding)
       
@@ -55,12 +55,13 @@ module EnsureValidEncoding
             c
           else
             options[:replace] || (
-             # surely there's a better way to tell if
-             # an encoding is a 'Unicode encoding form'
-             # than this? What's wrong with you ruby 1.9?
-             str.encoding.name.start_with?('UTF') ?
-               "\uFFFD".force_encoding(str.encoding) :
-               "?".force_encoding(str.encoding) )
+              # UTF-8 for unicode replacement char, encode in
+              # encoding of input string, using '?' as a fallback where
+              # it can't be (which should be non-unicode encodings)
+              "\uFFFD".force_encoding("UTF-8").encode( str.encoding, 
+                                                       :undef => :replace, 
+                                                       :replace => '?' )
+             )
           end
         end.join
       end

data/lib/ensure_valid_encoding/version.rb

@@ -1,3 +1,3 @@
 module EnsureValidEncoding
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

data/test/ensure_valid_encoding_test.rb

@@ -6,6 +6,7 @@ require 'ensure_valid_encoding'
 describe EnsureValidEncoding do
   before do
     @bad_bytes_utf8   = "M\xE9xico".force_encoding("UTF-8")  
+    @bad_bytes_utf16  = "M\x00\xDFxico".force_encoding( Encoding::UTF_16LE )
     @bad_bytes_ascii  = "M\xA1xico".force_encoding("ASCII")
   end
   
@@ -20,6 +21,11 @@ describe EnsureValidEncoding do
         :invalid => :replace).
       must_equal("M\uFFFDxico")
   end
+
+  it "replaces with unicode relacement string for UTF16LE" do
+    assert EnsureValidEncoding.ensure_valid_encoding(@bad_bytes_utf16,
+        :invalid => :replace).valid_encoding?
+  end
   
   it "replaces with chosen replacement string" do
     EnsureValidEncoding.ensure_valid_encoding(@bad_bytes_utf8, 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ensure_valid_encoding
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-10 00:00:00.000000000 Z
+date: 2012-07-13 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: