td 0.10.51 → 0.10.52

data/ChangeLog

@@ -1,4 +1,10 @@
 
+== 2012-09-26 version 0.10.52
+
+* bulk_import:upload_parts subcommand supports --parallel option to upload
+  files in parallel (default=2, max=8)
+
+
 == 2012-09-21 version 0.10.51
 
 * job:list and job:show show database name

data/lib/td/command/bulk_import.rb

@@ -104,6 +104,7 @@ module Command
     suffix_count = 0
     part_prefix = ""
     auto_perform = false
+    parallel = 2
 
     op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
       part_prefix = s
@@ -114,20 +115,53 @@ module Command
     op.on('--auto-perform', 'perform bulk import job automatically', TrueClass) {|b|
       auto_perform = b
     }
+    op.on('--parallel NUM', 'perform uploading in parallel (default: 2; max 8)', Integer) {|i|
+      parallel = i
+    }
 
     name, *files = op.cmd_parse
 
-    files.each {|ifname|
-      basename = File.basename(ifname)
-      part_name = part_prefix + basename.split('.')[0..suffix_count].join('.')
+    parallel = 1 if parallel <= 1
+    parallel = 8 if parallel >= 8
+
+    threads = (1..parallel).map {|i|
+      Thread.new do
+        errors = []
+        until files.empty?
+          ifname = files.shift
+          basename = File.basename(ifname)
+          begin
+            part_name = part_prefix + basename.split('.')[0..suffix_count].join('.')
+
+            File.open(ifname, "rb") {|io|
+              size = io.size
+              $stderr.write "Uploading '#{ifname}' -> '#{part_name}'... (#{size} bytes)\n"
+
+              bulk_import_upload_impl(name, part_name, io, size, retry_limit, retry_wait)
+            }
+          rescue
+            errors << [ifname, $!]
+          end
+        end
+        errors
+      end
+    }
 
-      File.open(ifname, "rb") {|io|
-        size = io.size
-        $stderr.puts "Uploading '#{ifname}' -> '#{part_name}'... (#{size} bytes)"
+    errors = []
+    threads.each {|t|
+      errors.concat t.value
+    }
 
-        bulk_import_upload_impl(name, part_name, io, size, retry_limit, retry_wait)
+    unless errors.empty?
+      $stderr.puts "failed to upload #{errors.size} files."
+      errors.each {|(ifname,ex)|
+        $stderr.puts "  #{ifname}: #{ex}"
+        ex.backtrace.each {|bt|
+          $stderr.puts "      #{ifname}: #{ex}"
+        }
       }
-    }
+      exit 1
+    end
 
     $stderr.puts "done."
 
@@ -311,6 +345,7 @@ module Command
       end
     }
 
+    # TODO multi process
     files.each {|ifname|
       $stderr.puts "Processing #{ifname}..."
       record_num = 0
@@ -361,7 +396,7 @@ module Command
     rescue
       if retry_limit > 0
         retry_limit -= 1
-        $stderr.puts "#{$!}; retrying '#{part_name}'..."
+        $stderr.write "#{$!}; retrying '#{part_name}'...\n"
         sleep retry_wait
         retry
       end

data/lib/td/command/list.rb

@@ -233,7 +233,7 @@ module List
   add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
   add_list 'bulk_import:prepare_parts', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_parts logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
   add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
-  add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --prefix logs_'
+  add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --parallel 4'
   add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
   add_list 'bulk_import:delete_parts', %w[name ids_], 'Delete uploaded files from a bulk import session', 'bulk_import:delete_parts logs_201201 01h 02h 03h'
   add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.51'
+VERSION = '0.10.52'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.51
+  version: 0.10.52
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-22 00:00:00.000000000 Z
+date: 2012-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack