td 0.10.39 → 0.10.40

data/ChangeLog

@@ -1,4 +1,11 @@
 
+== 2012-08-07 version 0.10.40
+
+* Added bulk_import:delete_parts
+* Hide obsoleted bulk_import:upload_part and :delete_part subcommands
+* bulk_import:upload_parts supports --auto-perform option
+
+
 == 2012-08-07 version 0.10.39
 
 * Added bulk_import:prepare_parts and bulk_import:upload_parts subcommands

data/lib/td/command/bulk_import.rb

@@ -84,6 +84,7 @@ module Command
     }
   end
 
+  # obsoleted
   def bulk_import_upload_part(op)
     retry_limit = 10
     retry_wait = 1
@@ -102,6 +103,7 @@ module Command
     retry_wait = 1
     suffix_count = 0
     part_prefix = ""
+    auto_perform = false
 
     op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
       part_prefix = s
@@ -109,6 +111,9 @@ module Command
     op.on('-s', '--use-suffix COUNT', 'use COUNT number of . (dots) in the source file name to the parts name', Integer) {|i|
       suffix_count = i
     }
+    op.on('--auto-perform', 'perform bulk import job automatically', TrueClass) {|b|
+      auto_perform = b
+    }
 
     name, *files = op.cmd_parse
 
@@ -125,8 +130,17 @@ module Command
     }
 
     $stderr.puts "done."
+
+    if auto_perform
+      client = get_client
+      job = client.perform_bulk_import(name)
+
+      $stderr.puts "Job #{job.job_id} is queued."
+      $stderr.puts "Use '#{$prog} job:show [-w] #{job.job_id}' to show the status."
+    end
   end
 
+  # obsoleted
   def bulk_import_delete_part(op)
     name, part_name = op.cmd_parse
 
@@ -137,6 +151,27 @@ module Command
     $stderr.puts "Part '#{part_name}' is deleted."
   end
 
+  def bulk_import_delete_parts(op)
+    part_prefix = ""
+
+    op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
+      part_prefix = s
+    }
+
+    name, *part_names = op.cmd_parse
+
+    client = get_client
+
+    part_names.each {|part_name|
+      part_name = part_prefix + part_name
+
+      $stderr.puts "Deleting '#{part_name}'..."
+      client.bulk_import_delete_part(name, part_name)
+    }
+
+    $stderr.puts "done."
+  end
+
   def bulk_import_perform(op)
     wait = false
     force = false
@@ -172,7 +207,7 @@ module Command
     job = client.perform_bulk_import(name)
 
     $stderr.puts "Job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} job:show [-w] #{job.job_id}' to show the status."
 
     if wait
       require 'td/command/job'  # wait_job
@@ -237,7 +272,7 @@ module Command
 
   PART_SPLIT_SIZE = 16*1024*1024
 
-  def bulk_import_prepare_part(op)
+  def bulk_import_prepare_parts(op)
     outdir = nil
     split_size_kb = PART_SPLIT_SIZE / 1024  # kb
 

data/lib/td/command/list.rb

@@ -128,6 +128,7 @@ module List
   COMMAND = {}
   GUESS = {}
   HELP_EXCLUDE = [/^help/, /^account/, /^aggr/]
+  USAGE_EXCLUDE = [/bulk_import:upload_part\z/, /bulk_import:delete_part\z/]
 
   def self.add_list(name, args, description, *examples)
     LIST << COMMAND[name] = CommandOption.new(name, args, description, examples)
@@ -201,7 +202,9 @@ module List
 
         msg = %[Additional commands, type "#{File.basename($0)} help COMMAND" for more details:\n\n]
         ops.each {|op|
-          msg << %[  #{op.usage}\n]
+          unless USAGE_EXCLUDE.any? {|pattern| pattern =~ op.name }
+            msg << %[  #{op.usage}\n]
+          end
         }
         msg << %[\n]
         c.override_message = msg
@@ -228,10 +231,11 @@ module List
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
   add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
-  add_list 'bulk_import:prepare_part', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_part logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
+  add_list 'bulk_import:prepare_parts', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_parts logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
   add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
   add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --prefix logs_'
   add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
+  add_list 'bulk_import:delete_parts', %w[name ids_], 'Delete uploaded files from a bulk import session', 'bulk_import:delete_parts logs_201201 01h 02h 03h'
   add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'
   add_list 'bulk_import:error_records', %w[name], 'Show records which did not pass validations', 'bulk_import:error_records logs_201201'
   add_list 'bulk_import:commit', %w[name], 'Start to commit a performed bulk import session', 'bulk_import:commit logs_201201'

data/lib/td/file_reader.rb

@@ -35,9 +35,6 @@ module TreasureData
         @reader = reader
         @delimiter_expr = opts[:delimiter_expr]
         @null_expr = opts[:null_expr]
-        # TODO
-        #@escape_char = opts[:escape_char]
-        #@quote_char = opts[:quote_char]
       end
 
       def next
@@ -49,6 +46,21 @@ module TreasureData
       end
     end
 
+    # TODO
+    #class QuotedDelimiterParsingReader
+    #  def initialize(io, error, opts)
+    #    require 'strscan'
+    #    @io = io
+    #    @error = error
+    #    @delimiter_expr = opts[:delimiter_expr]
+    #    @quote_char = opts[:quote_char]
+    #    @escape_char = opts[:escape_char]
+    #  end
+
+    #  def next
+    #  end
+    #end
+
     class JSONParser
       def initialize(reader, error, opts)
         @reader = reader
@@ -68,6 +80,7 @@ module TreasureData
       end
     end
 
+    # TODO
     #class ApacheParser
     #  REGEXP = /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/
     #
@@ -119,7 +132,6 @@ module TreasureData
 
       def next
         array = @parser.next
-        # FIXME error handling
         Hash[@columns.zip(array)]
       end
     end
@@ -190,22 +202,22 @@ module TreasureData
     attr_accessor :parser_class
 
     def init_optparse(op)
-      op.on('-f', '--format NAME', "source file format") {|s|
+      op.on('-f', '--format NAME', "source file format [csv, tsv, msgpack, json]") {|s|
         set_format_template(s)
       }
-      op.on('-h', '--columns NAME,NAME,...', 'column names') {|s|
+      op.on('-h', '--columns NAME,NAME,...', 'column names (use --column-header instead if the first line has column names)') {|s|
         @opts[:column_names] = s.split(',')
       }
       op.on('-H', '--column-header', 'first line includes column names', TrueClass) {|b|
         @opts[:column_header] = b
       }
-      op.on('-d', '--delimiter REGEX', "delimiter between columns (default: #{@default_opts[:delimiter_expr].inspect[1..-2]}") {|s|
+      op.on('-d', '--delimiter REGEX', "delimiter between columns (default: #{@default_opts[:delimiter_expr].inspect[1..-2]})") {|s|
         @opts[:delimiter_expr] = Regexp.new(s)
       }
       #op.on('-D', '--line-delimiter REGEX', "delimiter between rows (default: #{@default_opts[:line_delimiter_expr].inspect[1..-2]})") {|s|
       #  @opts[:line_delimiter_expr] = Regexp.new(s)
       #}
-      op.on('-N', '--null REGEX', "null expression (default: #{@default_opts[:null_expr].inspect[1..-2]}") {|s|
+      op.on('-N', '--null REGEX', "null expression (default: #{@default_opts[:null_expr].inspect[1..-2]})") {|s|
         @opts[:null_expr] = Regexp.new(s)
       }
       # TODO

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.39'
+VERSION = '0.10.40'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.39
+  version: 0.10.40
   prerelease: 
 platform: ruby
 authors: