td 0.10.52 → 0.10.53

data/ChangeLog

@@ -1,4 +1,11 @@
 
+== 2012-10-02 version 0.10.53
+
+* bulk_import:prepare_parts works with Ruby 1.8
+* bulk_import:prepare_parts supports 'msgpack' format
+* bulk_import:prepare_parts supports gzip decompression
+
+
 == 2012-09-26 version 0.10.52
 
 * bulk_import:upload_parts subcommand supports --parallel option to upload

data/lib/td/command/bulk_import.rb

@@ -321,7 +321,10 @@ module Command
       outdir = s
     }
 
-    *files = op.cmd_parse
+    files = op.cmd_parse
+
+    # TODO ruby 1.9
+    files = [files] unless files.is_a?(Array)
 
     unless outdir
       $stderr.puts "-o, --output DIR option is required."
@@ -350,7 +353,7 @@ module Command
       $stderr.puts "Processing #{ifname}..."
       record_num = 0
 
-      basename = File.basename(ifname).split('.').first
+      basename = File.basename(ifname).sub(/\.(?:csv|tsv|json|msgpack)(?:\.gz)?$/i,'').split('.').join('_')
       File.open(ifname) {|io|
         of_index = 0
         out = nil

data/lib/td/file_reader.rb

@@ -1,6 +1,28 @@
 
 module TreasureData
   class FileReader
+    require 'zlib'
+
+    class DecompressIOFilter
+      def self.filter(io, error, opts)
+        case opts[:compress]
+        when 'gzip'
+          return Zlib::GzipReader.new(io)
+        when 'plain'
+          return io
+        when nil
+          data = io.read(2)
+          io.rewind
+          if data.unpack('CC') == [0x1f, 0x8b]
+            return Zlib::GzipReader.new(io)
+          else
+            return io
+          end
+        else
+          raise "unknown compression type #{opts[:compress]}"
+        end
+      end
+    end
 
     class MessagePackParsingReader
       def initialize(io, error, opts)
@@ -11,7 +33,7 @@ module TreasureData
       end
 
       def forward
-        @u.forward
+        @u.each {|r| break r }
       end
     end
 
@@ -313,6 +335,7 @@ module TreasureData
       case @format
       when 'text'
         Proc.new {|io,error|
+          io = DecompressIOFilter.filter(io, error, opts)
           reader = LineReader.new(io, error, opts)
           parser = DelimiterParser.new(reader, error, opts)
           if opts[:column_header]
@@ -337,6 +360,7 @@ module TreasureData
 
       when 'json'
         Proc.new {|io,error|
+          io = DecompressIOFilter.filter(io, error, opts)
           reader = LineReader.new(io, error, opts)
           parser = JSONParser.new(reader, error, opts)
           if opts[:column_header]
@@ -356,6 +380,7 @@ module TreasureData
 
       when 'msgpack'
         Proc.new {|io,error|
+          io = DecompressIOFilter.filter(io, error, opts)
           parser = MessagePackParsingReader.new(io, error, opts)
           if opts[:column_header]
             column_names = parser.forward

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.52'
+VERSION = '0.10.53'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.52
+  version: 0.10.53
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-27 00:00:00.000000000 Z
+date: 2012-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack