td 0.10.31 → 0.10.32

data/ChangeLog

@@ -1,4 +1,10 @@
 
+== 2012-07-03 version 0.10.32
+
+* Added table:partial_delete subcommand
+* query, sched:create and sched:update subcommands support -P, --priority option
+
+
 == 2012-06-27 version 0.10.31
 
 * table:tail supports -P option which enable pretty printing

data/lib/td/command/job.rb

@@ -5,6 +5,27 @@ module Command
   # TODO
   JOB_WAIT_MAX_RETRY_COUNT_ON_NETWORK_ERROR = 10
 
+  PRIORITY_FORMAT_MAP = {
+    -2 => 'VERY LOW',
+    -1 => 'LOW',
+    0 => 'NORMAL',
+    1 => 'HIGH',
+    2 => 'VERY HIGH',
+  }
+
+  PRIORITY_PARSE_MAP = {
+    /\Avery[ _\-]?low\z/i => -2,
+    /\A-2\z/ => -2,
+    /\Alow\z/i => -1,
+    /\A-1\z/ => -1,
+    /\Anorm(?:al)?\z/i => 0,
+    /\A[\-\+]?0\z/ => 0,
+    /\Ahigh\z/i => 1,
+    /\A[\+]?1\z/ => 1,
+    /\Avery[ _\-]?high\z/i => 2,
+    /\A[\+]?2\z/ => 2,
+  }
+
   def job_list(op)
     page = 0
     skip = 0
@@ -41,10 +62,11 @@ module Command
     jobs.each {|job|
       start = job.start_at
       elapsed = cmd_format_elapsed(start, job.end_at)
-      rows << {:JobID => job.job_id, :Status => job.status, :Type => job.type, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.result_url}
+      priority = job_priority_name_of(job.priority)
+      rows << {:JobID => job.job_id, :Status => job.status, :Type => job.type, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Priority => priority, :Result => job.result_url}
     }
 
-    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed,:Result,  :Type, :Query])
+    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Priority, :Result, :Type, :Query])
   end
 
   def job_show(op)
@@ -79,8 +101,9 @@ module Command
     #puts "URL          : #{job.url}"
     puts "Status       : #{job.status}"
     puts "Type         : #{job.type}"
+    puts "Priority     : #{job_priority_name_of(job.priority)}"
+    puts "Result       : #{job.result_url}"
     puts "Query        : #{job.query}"
-    puts "Result table : #{job.result_url}"
 
     if wait && !job.finished?
       wait_job(job)
@@ -248,6 +271,17 @@ module Command
 
     puts cmd_render_table(rows, opts)
   end
+
+  def job_priority_name_of(id)
+    PRIORITY_FORMAT_MAP[id] || 'NORMAL'
+  end
+
+  def job_priority_id_of(name)
+    PRIORITY_PARSE_MAP.each_pair {|pattern,id|
+      return id if pattern.match(name)
+    }
+    return nil
+  end
 end
 end
 

data/lib/td/command/list.rb

@@ -223,6 +223,7 @@ module List
   add_list 'table:import', %w[db table files_], 'Parse and import files to a table', 'table:import example_db table1 --apache access.log', 'table:import example_db table1 --json -t time - < test.json'
   add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', 'table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY'
   add_list 'table:tail', %w[db table], 'Get recently imported logs', 'table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30'
+  add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', 'table:partial_delete example_db table1 --from 1341000000 --to 1341003600'
 
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'

data/lib/td/command/query.rb

@@ -10,6 +10,7 @@ module Command
     result_url = nil
     result_user = nil
     result_ask_password = false
+    priority = nil
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -35,6 +36,12 @@ module Command
     op.on('-p', '--password', 'ask password for the result URL') {|s|
       result_ask_password = true
     }
+    op.on('-P', '--priority PRIORITY', 'set priority') {|s|
+      priority = job_priority_id_of(s)
+      unless priority
+        raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
+      end
+    }
 
     sql = op.cmd_parse
 
@@ -53,7 +60,7 @@ module Command
     # local existance check
     get_database(client, db_name)
 
-    job = client.query(db_name, sql, result_url)
+    job = client.query(db_name, sql, result_url, priority)
 
     $stderr.puts "Job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
@@ -69,7 +76,7 @@ module Command
     end
   end
 
-  require 'td/command/job'  # wait_job
+  require 'td/command/job'  # wait_job, job_priority_id_of
 end
 end
 

data/lib/td/command/sched.rb

@@ -3,6 +3,8 @@ module TreasureData
 module Command
 
   def sched_list(op)
+    require 'td/command/job'  # job_priority_name_of
+
     op.cmd_parse
 
     client = get_client
@@ -11,13 +13,13 @@ module Command
 
     rows = []
     scheds.each {|sched|
-      rows << {:Name => sched.name, :Cron => sched.cron, :Timezone => sched.timezone, :Delay => sched.delay, :Result => sched.result_url, :Database => sched.database, :Query => sched.query, :"Next schedule" => sched.next_time ? sched.next_time.localtime : nil }
+      rows << {:Name => sched.name, :Cron => sched.cron, :Timezone => sched.timezone, :Delay => sched.delay, :Priority => job_priority_name_of(sched.priority), :Result => sched.result_url, :Database => sched.database, :Query => sched.query, :"Next schedule" => sched.next_time ? sched.next_time.localtime : nil }
     }
     rows = rows.sort_by {|map|
       map[:Name]
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Result, :Database, :Query], :max_width=>500)
+    puts cmd_render_table(rows, :fields => [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Priority, :Result, :Database, :Query], :max_width=>500)
   end
 
   def sched_create(op)
@@ -27,6 +29,7 @@ module Command
     result_url = nil
     result_user = nil
     result_ask_password = false
+    priority = nil
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -46,6 +49,12 @@ module Command
     op.on('-p', '--password', 'ask password for the result URL') {|s|
       result_ask_password = true
     }
+    op.on('-P', '--priority PRIORITY', 'set priority') {|s|
+      priority = job_priority_id_of(s)
+      unless priority
+        raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
+      end
+    }
 
     name, cron, sql = op.cmd_parse
 
@@ -65,7 +74,7 @@ module Command
     get_database(client, db_name)
 
     begin
-      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay)
+      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay, :priority=>priority)
     rescue AlreadyExistsError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' already exists."
@@ -99,6 +108,7 @@ module Command
     result = nil
     timezone = nil
     delay = nil
+    priority = nil
 
     op.on('-s', '--schedule CRON', 'change the schedule') {|s|
       cron = s
@@ -118,6 +128,12 @@ module Command
     op.on('-D', '--delay SECONDS', 'change the delay time of the schedule', Integer) {|i|
       delay = i
     }
+    op.on('-P', '--priority PRIORITY', 'set priority') {|s|
+      priority = job_priority_id_of(s)
+      unless priority
+        raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
+      end
+    }
 
     name = op.cmd_parse
 
@@ -128,6 +144,7 @@ module Command
     params['result'] = result if result
     params['timezone'] = timezone if timezone
     params['delay'] = delay.to_s if delay
+    params['priority'] = priority.to_s if priority
 
     if params.empty?
       $stderr.puts op.to_s
@@ -149,6 +166,8 @@ module Command
   end
 
   def sched_history(op)
+    require 'td/command/job'  # job_priority_name_of
+
     page = 0
     skip = 0
 
@@ -186,16 +205,17 @@ module Command
       puts "Delay        : #{s.delay} sec"
       puts "Next         : #{s.next_time}"
       puts "Result       : #{s.result_url}"
+      puts "Priority     : #{job_priority_name_of(s.priority)}"
       puts "Database     : #{s.database}"
       puts "Query        : #{s.query}"
     end
 
     rows = []
     history.each {|j|
-      rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Result=>j.result_url}
+      rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Priority => job_priority_name_of(j.priority), :Result=>j.result_url}
     }
 
-    puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Result])
+    puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result])
   end
 
   def sched_run(op)

data/lib/td/command/table.rb

@@ -134,7 +134,7 @@ module Command
 
     op.on('-t', '--to TIME', 'end time of logs to get') {|s|
       if s.to_i.to_s == s
-        to = s
+        to = s.to_i
       else
         require 'time'
         to = Time.parse(s).to_i
@@ -142,7 +142,7 @@ module Command
     }
     op.on('-f', '--from TIME', 'start time of logs to get') {|s|
       if s.to_i.to_s == s
-        from = s
+        from = s.to_i
       else
         require 'time'
         from = Time.parse(s).to_i
@@ -193,7 +193,6 @@ module Command
         puts row.to_json
       }
     end
-
   end
 
   def table_export(op)
@@ -254,6 +253,64 @@ module Command
     end
   end
 
+  def table_partial_delete(op)
+    from = nil
+    to = nil
+    wait = false
+
+    op.on('-t', '--to TIME', 'end time of logs to delete') {|s|
+      if s.to_i.to_s == s
+        # UNIX time
+        to = s.to_i
+      else
+        require 'time'
+        to = Time.parse(s).to_i
+      end
+    }
+    op.on('-f', '--from TIME', 'start time of logs to delete') {|s|
+      if s.to_i.to_s == s
+        from = s.to_i
+      else
+        require 'time'
+        from = Time.parse(s).to_i
+      end
+    }
+    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
+      wait = b
+    }
+
+    db_name, table_name = op.cmd_parse
+
+    unless from
+      $stderr.puts "-f, --from TIME option is required"
+      exit 1
+    end
+
+    unless to
+      $stderr.puts "-t, --to TIME option is required"
+      exit 1
+    end
+
+    if from % 3600 != 0 || to % 3600 != 0
+      $stderr.puts "time must be a multiple of 3600 (1 hour)"
+      exit 1
+    end
+
+    client = get_client
+
+    table = get_table(client, db_name, table_name)
+
+    job = client.partial_delete(db_name, table_name, to, from)
+
+    $stderr.puts "Partial delete job #{job.job_id} is queued."
+    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+
+    if wait && !job.finished?
+      wait_job(job)
+      puts "Status     : #{job.status}"
+    end
+  end
+
   require 'td/command/import'  # table:import
   require 'td/command/export'  # table:export
   require 'td/command/job'  # wait_job

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.31'
+VERSION = '0.10.32'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.31
+  version: 0.10.32
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-27 00:00:00.000000000Z
+date: 2012-07-03 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70192621206960 !ruby/object:Gem::Requirement
+  requirement: &70143943515280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70192621206960
+  version_requirements: *70143943515280
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70192621206320 !ruby/object:Gem::Requirement
+  requirement: &70143943514700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70192621206320
+  version_requirements: *70143943514700
 - !ruby/object:Gem::Dependency
   name: hirb
-  requirement: &70192621205780 !ruby/object:Gem::Requirement
+  requirement: &70143943514000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,21 +43,21 @@ dependencies:
         version: 0.4.5
   type: :runtime
   prerelease: false
-  version_requirements: *70192621205780
+  version_requirements: *70143943514000
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70192621205060 !ruby/object:Gem::Requirement
+  requirement: &70143943509200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.18
+        version: 0.8.19
   type: :runtime
   prerelease: false
-  version_requirements: *70192621205060
+  version_requirements: *70143943509200
 - !ruby/object:Gem::Dependency
   name: td-logger
-  requirement: &70192621204380 !ruby/object:Gem::Requirement
+  requirement: &70143943508660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 0.3.12
   type: :runtime
   prerelease: false
-  version_requirements: *70192621204380
+  version_requirements: *70143943508660
 description: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 email: support@treasure-data.com
 executables: