td 0.11.9 → 0.11.10

data/lib/td/command/sched.rb

@@ -1,5 +1,8 @@
+require 'td/command/options'
+
 module TreasureData
 module Command
+  include Options
 
   def sched_list(op)
     require 'td/command/job'  # job_priority_name_of
@@ -20,7 +23,7 @@ module Command
       map[:Name]
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Priority, :Result, :Database, :Query], :max_width=>500, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Priority, :Result, :Database, :Query], :max_width=>500, :render_format => op.render_format)
   end
 
   def sched_create(op)
@@ -218,9 +221,9 @@ module Command
     end
 
     if newname && curname != newname
-      puts "Schedule '#{curname}' is updated and its name changed to '#{newname}'."
+      $stdout.puts "Schedule '#{curname}' is updated and its name changed to '#{newname}'."
     else
-      puts "Schedule '#{curname}' is updated."
+      $stdout.puts "Schedule '#{curname}' is updated."
     end
   end
 
@@ -259,16 +262,16 @@ module Command
 
     scheds = client.schedules
     if s = scheds.find {|s| s.name == name }
-      puts "Name        : #{s.name}"
-      puts "Cron        : #{s.cron}"
-      puts "Timezone    : #{s.timezone}"
-      puts "Delay       : #{s.delay} sec"
-      puts "Next        : #{s.next_time}"
-      puts "Result      : #{s.result_url}"
-      puts "Priority    : #{job_priority_name_of(s.priority)}"
-      puts "Retry limit : #{s.retry_limit}"
-      puts "Database    : #{s.database}"
-      puts "Query       : #{s.query}"
+      $stdout.puts "Name        : #{s.name}"
+      $stdout.puts "Cron        : #{s.cron}"
+      $stdout.puts "Timezone    : #{s.timezone}"
+      $stdout.puts "Delay       : #{s.delay} sec"
+      $stdout.puts "Next        : #{s.next_time}"
+      $stdout.puts "Result      : #{s.result_url}"
+      $stdout.puts "Priority    : #{job_priority_name_of(s.priority)}"
+      $stdout.puts "Retry limit : #{s.retry_limit}"
+      $stdout.puts "Database    : #{s.database}"
+      $stdout.puts "Query       : #{s.query}"
     end
 
     rows = []
@@ -277,7 +280,7 @@ module Command
       rows << {:Time => scheduled_at, :JobID => j.job_id, :Status => j.status, :Priority => job_priority_name_of(j.priority), :Result=>j.result_url}
     }
 
-    puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result], :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result], :render_format => op.render_format)
   end
 
   def sched_run(op)
@@ -320,7 +323,86 @@ module Command
     }
 
     $stderr.puts "Scheduled #{num} jobs from #{t}."
-    puts cmd_render_table(rows, :fields => [:JobID, :Time], :max_width=>500, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:JobID, :Time], :max_width=>500, :render_format => op.render_format)
+  end
+
+  def sched_result(op)
+    options = job_show_options(op)
+    back_number = 1
+    op.on('--last [Number]', Integer, "show the result before N from the last. default: 1") do |n|
+      back_number = n ? n : 1
+    end
+
+    # save argv before calling cmd_parse, which removes flags from the argv array
+    argv_saved = op.argv.dup
+    name = op.cmd_parse
+
+    verbose     = options[:verbose]
+    wait        = options[:wait]
+    output      = options[:output]
+    format      = options[:format]
+    render_opts = options[:render_opts]
+    limit       = options[:limit]
+    exclude     = options[:exclude]
+
+    client = get_client
+    history = get_history(client, name, (back_number - 1), back_number)
+
+    job = history.first
+
+    if job.nil?
+      $stderr.puts "No jobs available for this query. Refer to 'sched:history'."
+      exit 1
+    end
+
+    # build the job:show command now
+    argv = job_show_option_argv(argv_saved, name, back_number)
+    argv << job.job_id
+
+    Runner.new.run(argv)
+  end
+
+  def job_show_option_argv(argv_saved, name, back_number)
+    argv = ['job:show']
+    argv += (argv_saved - [name]) if argv_saved.length > 0
+
+    # there are three argvs parters for sched_result.
+    # 1. without --last
+    # 2. --last (without Num)
+    # 3. --last Num
+    # 'back_number' is value of Num which was parsed by OptionParser.
+    # remove both "--last" and Num if they are.
+
+    index_of_last = argv.index("--last")
+
+    return argv unless index_of_last
+
+    index_of_next_of_last = index_of_last + 1
+
+    # the arg value following to "--last"
+    next_of_last = argv[index_of_next_of_last]
+
+    indexes_of_options_for_sched_result = [index_of_last]
+    indexes_of_options_for_sched_result << index_of_next_of_last if next_of_last == back_number.to_s
+
+    indexes_of_options_for_sched_result.each do |index|
+      argv[index] = nil
+    end
+
+    argv.compact
+  end
+
+  def get_history(client, name, from, to)
+    begin
+      history = client.history(name, from, to)
+    rescue NotFoundError
+      cmd_debug_error $!
+      $stderr.puts "Schedule '#{name}' does not exist."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "sched:list' to show list of the schedules."
+      exit 1
+    end
+
+    history
   end
 
 end # module Command

data/lib/td/command/schema.rb

@@ -8,11 +8,11 @@ module Command
     client = get_client
     table = get_table(client, db_name, table_name)
 
-    puts "#{db_name}.#{table_name} ("
+    $stdout.puts "#{db_name}.#{table_name} ("
     table.schema.fields.each {|f|
-      puts "  #{f.name}:#{f.type}"
+      $stdout.puts "  #{f.name}:#{f.type}"
     }
-    puts ")"
+    $stdout.puts ")"
   end
 
   def schema_set(op)

data/lib/td/command/server.rb

@@ -6,7 +6,7 @@ module Command
   def server_status(op)
     op.cmd_parse
 
-    puts Client.server_status
+    $stdout.puts Client.server_status
   end
 
   def server_endpoint(op)

data/lib/td/command/status.rb

@@ -57,9 +57,9 @@ module Command
     x4, y4 = status_render(x3+2, y3, "[Results]", results, :fields => [:Name, :URL])
 
     (y3-y4-1).times do
-      print "\eD"
+      $stdout.print "\eD"
     end
-    print "\eE"
+    $stdout.print "\eE"
   end
 
   private
@@ -69,13 +69,13 @@ module Command
     lines.unshift(msg)
     #lines.unshift("")
 
-    print "\e[#{movey}A" if movey > 0
+    $stdout.print "\e[#{movey}A" if movey > 0
 
     max_width = 0
     height = 0
     lines.each {|line|
-      print "\e[#{movex}C" if movex > 0
-      puts line
+      $stdout.print "\e[#{movex}C" if movex > 0
+      $stdout.puts line
       width = line.length
       max_width = width if max_width < width
       height += 1

data/lib/td/command/table.rb

@@ -191,7 +191,7 @@ module Command
     else
       fields = [:Database, :Table, :Type, :Count, :Size, 'Last import', 'Last log timestamp', :Schema]
     end
-    puts cmd_render_table(rows, :fields => fields, :max_width => 500, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => fields, :max_width => 500, :render_format => op.render_format)
 
     if rows.empty?
       if db_name
@@ -231,16 +231,16 @@ module Command
 
     table = get_table(client, db_name, table_name)
 
-    puts "Name        : #{table.db_name}.#{table.name}"
-    puts "Type        : #{table.type}"
-    puts "Count       : #{table.count}"
-    # p table.methods.each {|m| puts m}
-    puts "Primary key : #{table.primary_key}:#{table.primary_key_type}" if table.type == :item
-    puts "Schema      : ("
+    $stdout.puts "Name        : #{table.db_name}.#{table.name}"
+    $stdout.puts "Type        : #{table.type}"
+    $stdout.puts "Count       : #{table.count}"
+    # p table.methods.each {|m| $stdout.puts m}
+    $stdout.puts "Primary key : #{table.primary_key}:#{table.primary_key_type}" if table.type == :item
+    $stdout.puts "Schema      : ("
     table.schema.fields.each {|f|
-      puts "    #{f.name}:#{f.type}"
+      $stdout.puts "    #{f.name}:#{f.type}"
     }
-    puts ")"
+    $stdout.puts ")"
   end
 
   def table_tail(op)
@@ -288,11 +288,11 @@ module Command
         :space => ' '
       }
       rows.each {|row|
-        puts row.to_json(opts)
+        $stdout.puts row.to_json(opts)
       }
     else
       rows.each {|row|
-        puts row.to_json
+        $stdout.puts row.to_json
       }
     end
   end
@@ -301,6 +301,7 @@ module Command
     from = nil
     to = nil
     wait = false
+    pool_name = nil
 
     op.on('-t', '--to TIME', 'end time of logs to delete in Unix time multiple of 3600 (1 hour)',
                              '  or Ruby time string format (e.g. \'2014-07-01 14:00:00 JST\') where',
@@ -326,6 +327,9 @@ module Command
     op.on('-w', '--wait', 'wait for the job to finish', TrueClass) {|b|
       wait = b
     }
+    op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
+      pool_name = s
+    }
 
     db_name, table_name = op.cmd_parse
 
@@ -350,6 +354,7 @@ module Command
     table = get_table(client, db_name, table_name)
 
     opts = {}
+    opts['pool_name'] = pool_name if pool_name
     job = client.partial_delete(db_name, table_name, to, from, opts)
 
     $stderr.puts "Partial delete job #{job.job_id} is queued."
@@ -357,7 +362,7 @@ module Command
 
     if wait && !job.finished?
       wait_job(job)
-      puts "Status     : #{job.status}"
+      $stdout.puts "Status     : #{job.status}"
     end
   end
 
@@ -374,9 +379,9 @@ module Command
     client.update_expire(db_name, table_name, expire_days)
 
     if expire_days == 0
-      puts "Data expiration disabled for this table."
+      $stdout.puts "Data expiration disabled for this table."
     else
-      puts "Table set to expire data older than #{expire_days} days."
+      $stdout.puts "Table set to expire data older than #{expire_days} days."
     end
   end
 
@@ -488,12 +493,12 @@ module Command
     begin
       db = client.database(db_name)
     rescue ForbiddenError => e
-      puts "Warning: database and table validation skipped - #{e.message}"
+      $stdout.puts "Warning: database and table validation skipped - #{e.message}"
     else
       begin
         table = db.table(table_name)
       rescue ForbiddenError => e
-        puts "Warning: table validation skipped - #{e.message}"
+        $stdout.puts "Warning: table validation skipped - #{e.message}"
       end
     end
 
@@ -522,12 +527,12 @@ module Command
       import_log_file(file, path, client, db_name, table_name, parser)
     }
 
-    puts "done."
+    $stdout.puts "done."
   end
 
   private
   def import_log_file(file, path, client, db_name, table_name, parser)
-    puts "importing #{path}..."
+    $stdout.puts "importing #{path}..."
 
     out = Tempfile.new('td-import')
     out.binmode if out.respond_to?(:binmode)
@@ -549,17 +554,17 @@ module Command
       n += 1
       x += 1
       if n % 10000 == 0   # by records imported
-        puts "  imported #{n} entries from #{path}..."
+        $stdout.puts "  imported #{n} entries from #{path}..."
 
       # TODO size
       elsif out.pos > 1024 * 1024   # by 1 MB chunks
-        puts "  imported #{n} entries from #{path}..."
+        $stdout.puts "  imported #{n} entries from #{path}..."
         begin
           writer.finish
           size = out.pos
           out.pos = 0
 
-          puts "  uploading #{size} bytes..."
+          $stdout.puts "  uploading #{size} bytes..."
           client.import(db_name, table_name, "msgpack.gz", out, size)
 
           out.truncate(0)
@@ -579,7 +584,7 @@ module Command
       size = out.pos
       out.pos = 0
 
-      puts "  uploading #{size} bytes..."
+      $stdout.puts "  uploading #{size} bytes..."
       # TODO upload on background thread
       client.import(db_name, table_name, "msgpack.gz", out, size)
     end
@@ -589,7 +594,7 @@ module Command
       raise ImportError, "no valid record to import from #{path}"
     end
 
-    puts "  imported #{n} entries from #{path}."
+    $stdout.puts "  imported #{n} entries from #{path}."
     $stderr.puts normalized_message if has_bignum
   ensure
     out.close rescue nil

data/lib/td/command/update.rb

@@ -12,11 +12,11 @@ module Command
     end
 
     start_time = Time.now
-    puts "Updating 'td' from #{TOOLBELT_VERSION}..."
+    $stdout.puts "Updating 'td' from #{TOOLBELT_VERSION}..."
     if new_version = Updater.update
-      puts "Successfully updated to #{new_version} in #{Command.humanize_time((Time.now - start_time).to_i)}."
+      $stdout.puts "Successfully updated to #{new_version} in #{Command.humanize_time((Time.now - start_time).to_i)}."
     else
-      puts "Nothing to update."
+      $stdout.puts "Nothing to update."
     end
   end
 

data/lib/td/command/user.rb

@@ -33,7 +33,7 @@ module Command
       rows << {:Name => user.name, :Email => user.email}
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Email], :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Name, :Email], :render_format => op.render_format)
 
     if rows.empty?
       $stderr.puts "There are no users."
@@ -73,13 +73,13 @@ module Command
       1.times { r << symbol.sort_by{rand}.first }
       password = r.sort_by{rand}.join
 
-      puts "Password: #{password}"
+      $stdout.puts "Password: #{password}"
 
     else
       3.times do
         begin
           system "stty -echo"  # TODO termios
-          print "Password (typing will be hidden): "
+          $stdout.print "Password (typing will be hidden): "
           password = STDIN.gets || ""
           password = password[0..-2]  # strip \n
         rescue Interrupt
@@ -87,7 +87,7 @@ module Command
           exit 1
         ensure
           system "stty echo"   # TODO termios
-          print "\n"
+          $stdout.print "\n"
         end
 
         if password.empty?
@@ -97,7 +97,7 @@ module Command
 
         begin
           system "stty -echo"  # TODO termios
-          print "Retype password: "
+          $stdout.print "Retype password: "
           password2 = STDIN.gets || ""
           password2 = password2[0..-2]  # strip \n
         rescue Interrupt
@@ -105,14 +105,14 @@ module Command
           exit 1
         ensure
           system "stty echo"   # TODO termios
-          print "\n"
+          $stdout.print "\n"
         end
 
         if password == password2
           break
         end
 
-        puts "Doesn't match."
+        $stdout.puts "Doesn't match."
       end
     end
 
@@ -185,7 +185,7 @@ module Command
       rows << {:Key => key}
     }
 
-    puts cmd_render_table(rows, :fields => [:Key], :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Key], :render_format => op.render_format)
   end
 
   def user_password_change(op)
@@ -196,7 +196,7 @@ module Command
     3.times do
       begin
         system "stty -echo"  # TODO termios
-        print "New password (typing will be hidden): "
+        $stdout.print "New password (typing will be hidden): "
         password = STDIN.gets || ""
         password = password[0..-2]  # strip \n
       rescue Interrupt
@@ -204,7 +204,7 @@ module Command
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
 
       if password.empty?
@@ -214,7 +214,7 @@ module Command
 
       begin
         system "stty -echo"  # TODO termios
-        print "Retype new password: "
+        $stdout.print "Retype new password: "
         password2 = STDIN.gets || ""
         password2 = password2[0..-2]  # strip \n
       rescue Interrupt
@@ -222,14 +222,14 @@ module Command
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
 
       if password == password2
         break
       end
 
-      puts "Doesn't match."
+      $stdout.puts "Doesn't match."
     end
 
     client = get_client(:ssl => true)

data/lib/td/helpers.rb

@@ -3,12 +3,7 @@ module TreasureData
     module_function
 
     def format_with_delimiter(number, delimiter = ',')
-      num = number.to_s
-      if formatted = num.gsub!(/(\d)(?=(?:\d{3})+(?!\d))/, "\\1#{delimiter}")
-        formatted
-      else
-        num
-      end
+      number.to_s.gsub(/(\d)(?=(?:\d{3})+(?!\d))/, "\\1#{delimiter}")
     end
 
     def home_directory

data/lib/td/updater.rb

@@ -166,14 +166,14 @@ module ModuleDefinition
           # downloading the update compressed file
           File.open("#{download_dir}/td-update.zip", "wb") do |file|
             endpoint = update_package_endpoint
-            puts "\npackage '#{endpoint}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
+            $stdout.puts "\npackage '#{endpoint}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
             stream_fetch(endpoint, file) {
               indicator.update
             }
           end
           indicator.finish
 
-          print "Unpacking updated toolbelt package..."
+          $stdout.print "Unpacking updated toolbelt package..."
           Zip::ZipFile.open("#{download_dir}/td-update.zip") do |zip|
             zip.each do |entry|
               target = File.join(download_dir, entry.to_s)
@@ -181,7 +181,7 @@ module ModuleDefinition
               zip.extract(entry, target) { true }
             end
           end
-          print "done\n"
+          $stdout.print "done\n"
 
           FileUtils.rm "#{download_dir}/td-update.zip"
 
@@ -270,7 +270,7 @@ module ModuleDefinition
 
     http.request_get(uri.path) {|response|
       if response.class == Net::HTTPOK
-        # print a . every tick_period seconds
+        # $stdout.print a . every tick_period seconds
         response.read_body do |chunk|
           binfile.write chunk
           progress.call unless progress.nil?
@@ -279,7 +279,7 @@ module ModuleDefinition
       elsif response.class == Net::HTTPFound || \
             response.class == Net::HTTPRedirection
         unless ENV['TD_TOOLBELT_DEBUG'].nil?
-          puts "redirect '#{url}' to '#{response['Location']}'... "
+          $stdout.puts "redirect '#{url}' to '#{response['Location']}'... "
         end
         return stream_fetch(response['Location'], binfile, &progress)
       else
@@ -353,15 +353,15 @@ end # module ModuleDefinition
         indicator.finish()
 
         if status
-          puts "Installed td-import.jar v#{version} in '#{Updater.jarfile_dest_path}'.\n"
+          $stdout.puts "Installed td-import.jar v#{version} in '#{Updater.jarfile_dest_path}'.\n"
           File.rename 'td-import.jar.new', 'td-import.jar'
         else
-          puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+          $stdout.puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
           File.delete 'td-import.jar.new' if File.exists? 'td-import.jar.new'
         end
       end
     else
-      puts 'Installed td-import.jar is already at the latest version.' unless hourly
+      $stdout.puts 'Installed td-import.jar is already at the latest version.' unless hourly
     end
   end
 
@@ -370,7 +370,7 @@ end # module ModuleDefinition
       if !ENV['TD_TOOLBELT_JAR_UPDATE'].nil?
         # also validates the TD_TOOLBELT_JAR_UPDATE environment variable value
         if ENV['TD_TOOLBELT_JAR_UPDATE'] == "0"
-          puts "Warning: Bulk Import JAR auto-update disabled by TD_TOOLBELT_JAR_UPDATE=0"
+          $stdout.puts "Warning: Bulk Import JAR auto-update disabled by TD_TOOLBELT_JAR_UPDATE=0"
           return
         elsif ENV['TD_TOOLBELT_JAR_UPDATE'] != "1"
           raise UpdateError,

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.11.9'
+  TOOLBELT_VERSION = '0.11.10'
 end

data/spec/td/command/connector_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'td/command/common'
+require 'td/command/list'
+require 'td/command/connector'
+
+module TreasureData::Command
+  describe 'connector commands' do
+    describe '#connector_preview' do
+      subject do
+        backup = $stdout.dup
+        buf = StringIO.new
+
+        begin
+          $stdout = buf
+
+          TreasureData::Command::Runner.new.run ["connector:preview", tempfile]
+
+          buf.string
+        ensure
+          $stdout = backup
+        end
+      end
+
+      let(:tempfile) do
+        File.join("spec", "td", "fixture", "bulk_load.yml")
+      end
+
+      let(:preview_result) do
+        {
+          "schema" => [
+            {"index" => 0, "name" => "c0_too_l#{'o' * 60}ng_column_name", "type" => "string"},
+            {"index" => 1, "name" => "c1", "type" => "long"},
+            {"index" => 2, "name" => "c2", "type" => "string"},
+            {"index" => 3, "name" => "c3", "type" => "string"}
+          ],
+          "records" => [
+            ["19920116", 32864, "06612", "00195"],
+            ["19910729", 14824, "07706", "00058"],
+            ["19881022", 26114, "06960", "00175"]
+          ]
+        }
+      end
+
+      before do
+        TreasureData::Client.any_instance.stub(:bulk_load_preview).and_return(preview_result)
+      end
+
+      it 'should include too_long_column_name without truncated' do
+        too_long_column_name = preview_result["schema"][0]["name"]
+        expect(subject).to include "#{too_long_column_name}:string"
+      end
+    end
+  end
+end