td 0.10.23 → 0.10.24

data/ChangeLog

@@ -1,4 +1,20 @@
 
+== 2012-05-10 version 0.10.24
+
+* Added bulk_import subcommands
+  * bulk_import:list
+  * bulk_import:show <name>
+  * bulk_import:create <name> <db> <table>
+  * bulk_import:upload_part <name> <id> <path.msgpack.gz>
+  * bulk_import:delete_part <name> <id>
+  * bulk_import:perform <name>
+  * bulk_import:error_records <name>
+  * bulk_import:commit <name>
+  * bulk_import:delete <name>
+  * bulk_import:freeze <name>
+  * bulk_import:unfreeze <name>
+
+
 == 2012-04-26 version 0.10.23
 
 * Added result:list, result:create and result:delete subcommands

data/lib/td/command/bulk_import.rb

@@ -0,0 +1,205 @@
+
+module TreasureData
+module Command
+
+  def bulk_import_list(op)
+    op.cmd_parse
+
+    client = get_client
+
+    bis = client.bulk_imports
+
+    rows = []
+    bis.each {|bi|
+      rows << {:Name=>bi.name, :Table=>"#{bi.database}.#{bi.table}", :Status=>bi.status.to_s.capitalize, :Frozen=>bi.upload_frozen? ? 'frozen' : '', :JobID=>bi.job_id, :"Valid Parts"=>bi.valid_parts, :"Error Parts"=>bi.error_parts, :"Valid Records"=>bi.valid_records, :"Error Records"=>bi.error_records}
+    }
+
+    puts cmd_render_table(rows, :fields => [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"])
+
+    if rows.empty?
+      $stderr.puts "There are no bulk import sessions."
+      $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
+    end
+  end
+
+  def bulk_import_create(op)
+    name, db_name, table_name = op.cmd_parse
+
+    client = get_client
+
+    table = get_table(client, db_name, table_name)
+
+    client.create_bulk_import(name, db_name, table_name)
+
+    $stderr.puts "Bulk import session '#{name}' is created."
+  end
+
+  def bulk_import_delete(op)
+    name = op.cmd_parse
+
+    client = get_client
+
+    begin
+      client.delete_bulk_import(name)
+    rescue NotFoundError
+      cmd_debug_error $!
+      $stderr.puts "Bulk import session '#{name}' does not exist."
+      exit 1
+    end
+
+    $stderr.puts "Bulk import session '#{name}' is deleted."
+  end
+
+  def bulk_import_show(op)
+    name = op.cmd_parse
+
+    client = get_client
+
+    bis = client.bulk_imports
+    bi = bis.find {|bi| name == bi.name }
+    unless bi
+      $stderr.puts "Bulk import session '#{name}' does not exist."
+      $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
+      exit 1
+    end
+
+    $stderr.puts "Name         : #{bi.name}"
+    $stderr.puts "Database     : #{bi.database}"
+    $stderr.puts "Table        : #{bi.table}"
+    $stderr.puts "Status       : #{bi.status.to_s.capitalize}"
+    $stderr.puts "Frozen       : #{bi.upload_frozen?}"
+    $stderr.puts "JobID        : #{bi.job_id}"
+    $stderr.puts "Valid Records: #{bi.valid_records}"
+    $stderr.puts "Error Records: #{bi.error_records}"
+    $stderr.puts "Valid Parts  : #{bi.valid_parts}"
+    $stderr.puts "Error Parts  : #{bi.error_parts}"
+    $stderr.puts "Uploaded Parts :"
+
+    list = client.list_bulk_import_parts(name)
+    list.each {|name|
+      puts "  #{name}"
+    }
+  end
+
+  def bulk_import_upload_part(op)
+    name, part_name, path = op.cmd_parse
+
+    client = get_client
+
+    File.open(path) {|is|
+      client.bulk_import_upload_part(name, part_name, is, is.size)
+    }
+
+    $stderr.puts "Part '#{part_name}' is uploaded."
+  end
+
+  def bulk_import_delete_part(op)
+    name, part_name = op.cmd_parse
+
+    client = get_client
+
+    client.bulk_import_delete_part(name, part_name)
+
+    $stderr.puts "Part '#{part_name}' is deleted."
+  end
+
+  def bulk_import_perform(op)
+    wait = false
+    force = false
+
+    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
+      wait = b
+    }
+    op.on('-f', '--force', 'force start performing', TrueClass) {|b|
+      force = b
+    }
+
+    name = op.cmd_parse
+
+    client = get_client
+
+    unless force
+      bis = client.bulk_imports
+      bi = bis.find {|bi| name == bi.name }
+      if bi
+        if bi.status == 'performing'
+          $stderr.puts "Bulk import session '#{name}' is already performing."
+          $stderr.puts "Add '-f' option to force start."
+          $stderr.puts "Use '#{$prog} job:kill #{bi.job_id}' to cancel the last trial."
+          exit 1
+        elsif bi.status == 'ready'
+          $stderr.puts "Bulk import session '#{name}' is already ready to commit."
+          $stderr.puts "Add '-f' option to force start."
+          exit 1
+        end
+      end
+    end
+
+    job = client.perform_bulk_import(name)
+
+    $stderr.puts "Job #{job.job_id} is queued."
+    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+
+    if wait
+      require 'td/command/job'  # wait_job
+      wait_job(job)
+    end
+  end
+
+  def bulk_import_commit(op)
+    name = op.cmd_parse
+
+    client = get_client
+
+    job = client.commit_bulk_import(name)
+
+    $stderr.puts "Bulk import session '#{name}' started to commit."
+  end
+
+  def bulk_import_error_records(op)
+    name = op.cmd_parse
+
+    client = get_client
+
+    bis = client.bulk_imports
+    bi = bis.find {|bi| name == bi.name }
+    unless bi
+      $stderr.puts "Bulk import session '#{name}' does not exist."
+      $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
+      exit 1
+    end
+
+    if bi.status == "uploading" || bi.status == "performing"
+      $stderr.puts "Bulk import session '#{name}' is not performed."
+      $stderr.puts "Use '#{$prog} bulk_import:perform <name>' to run."
+      exit 1
+    end
+
+    client.bulk_import_error_records(name) {|r|
+      puts r.to_json
+    }
+  end
+
+  def bulk_import_freeze(op)
+    name = op.cmd_parse
+
+    client = get_client
+
+    client.freeze_bulk_import(name)
+
+    $stderr.puts "Bulk import session '#{name}' is frozen."
+  end
+
+  def bulk_import_unfreeze(op)
+    name = op.cmd_parse
+
+    client = get_client
+
+    client.unfreeze_bulk_import(name)
+
+    $stderr.puts "Bulk import session '#{name}' is unfrozen."
+  end
+
+end
+end
+

data/lib/td/command/list.rb

@@ -224,6 +224,18 @@ module List
   add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', 'table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY'
   add_list 'table:tail', %w[db table], 'Get recently imported logs', 'table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30'
 
+  add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
+  add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
+  add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
+  add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
+  add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
+  add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'
+  add_list 'bulk_import:error_records', %w[name], 'Show records which did not pass validations', 'bulk_import:error_records logs_201201'
+  add_list 'bulk_import:commit', %w[name], 'Start to commit a performed bulk import session', 'bulk_import:commit logs_201201'
+  add_list 'bulk_import:delete', %w[name], 'Delete a bulk import session', 'bulk_import:delete logs_201201'
+  add_list 'bulk_import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', 'bulk_import:freeze logs_201201'
+  add_list 'bulk_import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'bulk_import:unfreeze logs_201201'
+
   add_list 'result:list', %w[], 'Show list of result URLs', 'result:list', 'results'
   add_list 'result:show', %w[name], 'Describe information of a result URL', 'result mydb'
   add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create mydb mysql://my-server/mydb'
@@ -298,6 +310,9 @@ module List
   add_alias 'scheds', 'sched:list'
   add_alias 'schedules', 'sched:list'
 
+  add_alias 'bulk_import', 'bulk_import:show'
+  add_alias 'bulk_imports', 'bulk_import:list'
+
   add_alias 'job', 'job:show'
   add_alias 'jobs', 'job:list'
   add_alias 'kill', 'job:kill'

data/lib/td/command/table.rb

@@ -98,13 +98,13 @@ module Command
     if rows.empty?
       if db_name
         $stderr.puts "Database '#{db_name}' has no tables."
-        $stderr.puts "Use '#{$prog} table:create <db.table>' to create a table."
+        $stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
       elsif dbs.empty?
         $stderr.puts "There are no databases."
         $stderr.puts "Use '#{$prog} db:create <db>' to create a database."
       else
         $stderr.puts "There are no tables."
-        $stderr.puts "Use '#{$prog} table:create <db.table>' to create a table."
+        $stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
       end
     end
   end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.23'
+VERSION = '0.10.24'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.23
+  version: 0.10.24
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-27 00:00:00.000000000Z
+date: 2012-05-10 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70287442084840 !ruby/object:Gem::Requirement
+  requirement: &70362122344260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70287442084840
+  version_requirements: *70362122344260
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70287442084360 !ruby/object:Gem::Requirement
+  requirement: &70362122343780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70287442084360
+  version_requirements: *70362122343780
 - !ruby/object:Gem::Dependency
   name: hirb
-  requirement: &70287442083880 !ruby/object:Gem::Requirement
+  requirement: &70362122343300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,21 +43,21 @@ dependencies:
         version: 0.4.5
   type: :runtime
   prerelease: false
-  version_requirements: *70287442083880
+  version_requirements: *70362122343300
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70287442083400 !ruby/object:Gem::Requirement
+  requirement: &70362122342820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.15
+        version: 0.8.16
   type: :runtime
   prerelease: false
-  version_requirements: *70287442083400
+  version_requirements: *70362122342820
 - !ruby/object:Gem::Dependency
   name: td-logger
-  requirement: &70287442082860 !ruby/object:Gem::Requirement
+  requirement: &70362122342340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 0.3.8
   type: :runtime
   prerelease: false
-  version_requirements: *70287442082860
+  version_requirements: *70362122342340
 description: 
 email: 
 executables:
@@ -81,6 +81,7 @@ files:
 - lib/td/command/account.rb
 - lib/td/command/aggr.rb
 - lib/td/command/apikey.rb
+- lib/td/command/bulk_import.rb
 - lib/td/command/common.rb
 - lib/td/command/db.rb
 - lib/td/command/export.rb