td 0.10.53 → 0.10.54

data/ChangeLog

@@ -1,4 +1,10 @@
 
+== 2012-10-04 version 0.10.54
+
+* check invalid byte sequence in UTF-8 string before rendering results on
+console
+
+
 == 2012-10-02 version 0.10.53
 
 * bulk_import:prepare_parts works with Ruby 1.8

data/lib/td/command/job.rb

@@ -260,13 +260,17 @@ module Command
     job.result_each {|row|
       # TODO limit number of rows to show
       rows << row.map {|v|
-        # TODO encoding check
         if v.is_a?(String)
           s = v.to_s
         else
           s = v.to_json
         end
-        s.force_encoding('UTF-8') if s.respond_to?(:force_encoding)
+        # Here does UTF-8 -> UTF-16LE -> UTF8 conversion:
+        #   a) to make sure the string doesn't include invalid byte sequence
+        #   b) to display multi-byte characters as it is
+        #   c) encoding from UTF-8 to UTF-8 doesn't check/replace invalid chars
+        #   d) UTF-16LE was slightly faster than UTF-16BE, UTF-32LE or UTF-32BE
+        s = s.encode('UTF-16LE', 'UTF-8', :invalid=>:replace, :undef=>:replace).encode('UTF-8') if s.respond_to?(:encode)
         s
       }
     }

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.53'
+VERSION = '0.10.54'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.53
+  version: 0.10.54
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-03 00:00:00.000000000 Z
+date: 2012-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack