td 0.11.7 → 0.11.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4b7f5341cbcebd3487c7df38951a4275204507e
-  data.tar.gz: 9aad2c3285c7beb623a97410b62844ff41f74574
+  metadata.gz: 7517598e29ec516a589d42d0b1955ddbe2279493
+  data.tar.gz: aa7597906e87e24fe94c9529bd33eede0922a218
 SHA512:
-  metadata.gz: 24846c57a7df595675ca6999412b0c542ac7beb1850adda09de6f9832152954d33ff2be5335b293213f92e492e558ab711f132598587433b0249a4f803eedebc
-  data.tar.gz: 65eed33f8222fe6c3fb57067087d1cc477fb3a9bc64d9d427b0375964281c32726b11f8b65a0258ab77ce914a7530a0aadfaa92adb0d40ae243b5a63369294d9
+  metadata.gz: b59f3192dece017a26edd4012f55c02ab54cc135c7e63d054fa8e7eeec259f43faf59f0b20a1207f7b30e48064dc00735a1d2df8bd84e8540fb0d4a3a965e495
+  data.tar.gz: af178fd2dc244ba7726e7ad5c62a8be7dc6bec3b5f8a6bf357cfb2ddf432b3cf302f7ba2b91d6e83e4173effce885c4a2be0d409a0b712d61fafe2d651a6c755

data/ChangeLog

@@ -1,3 +1,23 @@
+== 2015-02-20 version 0.11.8
+
+* Fix: save CSV/TSV file without transcoding.
+
+  The fix in 9d74b44b533878ee5f2a84db3506279f87f85bef at 0.11.7 broke the
+  job:show command when the download format is CSV or TSV and the target
+  query result contains non UTF-8 encoded byte sequence.
+
+  0.11.6 and before does transcoding and *sanitizes* non UTF-8 byte
+  sequence before saving to file. So the generated file is always in
+  UTF-8, but could be broken when query result is not in UTF-8.
+
+  0.11.7 simply removes that transcoding logic by
+  9d74b44b533878ee5f2a84db3506279f87f85bef for performance and crashes for
+  non UTF-8 encoded byte sequence.
+
+  This release fixes 0.11.7 and saves non UTF-8 encoded byte sequence
+  as-is (through 'BINARY' encoding) so that the generated file should be
+  properly encoded in UTF-8, Shift_JIS, or whatever.
+
 == 2015-02-16 version 0.11.7
 
 * Fix server:endpoint command not working

data/lib/td/command/job.rb

@@ -515,7 +515,10 @@ module Command
   end
 
   def dump_column(v)
-    v.is_a?(String) ? v.to_s : Yajl.dump(v)
+    s = v.is_a?(String) ? v.to_s : Yajl.dump(v)
+    # CAUTION: msgpack-ruby populates byte sequences as Encoding.default_internal which should be BINARY
+    s = s.force_encoding('BINARY') if s.respond_to?(:encode)
+    s
   end
 
   def dump_column_safe_utf8(v)

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.11.7'
+  TOOLBELT_VERSION = '0.11.8'
 end

data/spec/td/command/job_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'spec_helper'
 require 'td/command/common'
 require 'td/command/job'
@@ -10,17 +12,17 @@ module TreasureData::Command
       Class.new { include TreasureData::Command }.new
     end
 
-    let :job do
-      job = TreasureData::Job.new(nil, 12345, 'hive', 'select * from employee')
-      job.instance_eval do
-        @result = [[["1", 2.0, {key:3}], 1], [["4", 5.0, {key:6}], 2], [["7", 8.0, {key:9}], 3]]
-        @result_size = 3
-        @status = 'success'
+    describe 'write_result' do
+      let :job do
+        job = TreasureData::Job.new(nil, 12345, 'hive', 'select * from employee')
+        job.instance_eval do
+          @result = [[["1", 2.0, {key:3}], 1], [["4", 5.0, {key:6}], 2], [["7", 8.0, {key:9}], 3]]
+          @result_size = 3
+          @status = 'success'
+        end
+        job
       end
-      job
-    end
 
-    describe 'write_result' do
       it 'supports json output' do
         file = Tempfile.new("job_spec")
         command.send(:show_result, job, file, nil, 'json')
@@ -39,5 +41,62 @@ module TreasureData::Command
         File.read(file.path).should == %Q(1\t2.0\t{"key":3}\n4\t5.0\t{"key":6}\n7\t8.0\t{"key":9}\n)
       end
     end
+
+    describe 'multibyte chars' do
+      let :multibyte_string do
+        # Originally a Windows-31J but in UTF-8 like msgpack-ruby populates
+        "\x83\x81\x81[\x83\x8B"
+      end
+
+      let :multibyte_row do
+        [multibyte_string, 2.0, {multibyte_string => multibyte_string}]
+      end
+
+      let :job do
+        row = multibyte_row
+        job = TreasureData::Job.new(nil, 12345, 'hive', 'select * from employee')
+        job.instance_eval do
+          @result = [[row, 1], [row, 2]]
+          @result_size = 2
+          @status = 'success'
+        end
+        job
+      end
+
+      it 'assumes test setting is correct' do
+        # the String is actually in Windows-31J but encoding is UTF-8 msgpack-ruby reports
+        multibyte_string.encoding.should == Encoding::UTF_8
+        multibyte_string.force_encoding('Windows-31J').encode('UTF-8').should == 'メール'
+      end
+
+      it 'supports json output' do
+        row = multibyte_row
+        file = Tempfile.new("job_spec")
+        command.send(:show_result, job, file, nil, 'json')
+        File.read(file.path).should == '[' + [row, row].map { |e| Yajl.dump(e) }.join(",\n") + ']'
+      end
+
+      it 'supports csv output' do
+        row = multibyte_row.map { |e| dump_column(e) }
+        file = Tempfile.new("job_spec")
+        command.send(:show_result, job, file, nil, 'csv')
+        File.binread(file.path).should == [row, row].map { |e| CSV.generate_line(e) }.join
+        File.open(file.path, 'r:Windows-31J').read.encode('UTF-8').split.first.should == 'メール,2.0,"{""メール"":""メール""}"'
+      end
+
+      it 'supports tsv output' do
+        row = multibyte_row.map { |e| dump_column(e) }
+        file = Tempfile.new("job_spec")
+        command.send(:show_result, job, file, nil, 'tsv')
+        File.binread(file.path).should == [row, row].map { |e| e.join("\t") + "\n" }.join
+        File.open(file.path, 'r:Windows-31J').read.encode('UTF-8').split("\n").first.should == "メール\t2.0\t{\"メール\":\"メール\"}"
+      end
+    end
+
+    def dump_column(v)
+      s = v.is_a?(String) ? v.to_s : Yajl.dump(v)
+      s = s.force_encoding('BINARY')
+      s
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.11.7
+  version: 0.11.8
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-17 00:00:00.000000000 Z
+date: 2015-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack