td 0.10.99 → 0.11.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MjhhYWI5NWE1ZmI3ZTkwZjY0OTNhMGE3N2VjNjUzNzk2ZWY2OGJkYQ==
+  data.tar.gz: !binary |-
+    NDZmMDJlMDQyZmViMjY5NTNmMTE0ODg5ZWE2MGYxZjJhYTg4Y2VhZQ==
+SHA512:
+  metadata.gz: !binary |-
+    NGZmMTk4ZTk5ZWFhYzc4YmEyZTE0ZmI0MjA2ODYzOWQxNzQ0ZTc5OGRjZDhh
+    OTI0NjJiZjA3MDhiZmFjNjg3MThjNjI1ZjcyZjNiOTNiOWU5MjQ2M2FhOWRh
+    ZTI4OGI2ZTk1YTEwYmVkZGJmNGU4Y2YzYTY1ZDc5MmNkYTI5ZTQ=
+  data.tar.gz: !binary |-
+    NjNiNTI4MGI4MDY0MGYzMjZiMTBmNzBhYzJmNjNmZjcxZDgxZWVjNTgzYzUx
+    ZGZiMzViYmQ0ZjQwNWIwZmI5MWQ5ZmExYTA3MzZkZDlkY2RmNTEzNDMxYzNj
+    ZGJhZTFlZDExYjc4YzdkYzUwZmJjNjc3NTlkZGQ5OTI2MDQ4NjY=

data/.gitignore

@@ -1,5 +1,8 @@
 .bundle
+.DS_Store
 build/td-import-java
 Gemfile.lock
 vendor/*
+pkg/
+build/*
 *~

data/ChangeLog

@@ -1,10 +1,39 @@
+== 2014-04-29 version 0.11.1
+
+* Fix Treasure Data query result output database and table validation
+  regular expression
+* Fixed auto updater which threw a warning due to a constant being
+  reinitialized
+* Interpret the CPU time as number of milliseconds as opposed to
+  number of seconds in the job:show output
+* when outputting the query result to file, flush the data every
+  100 records
+
+== 2014-04-23 version 0.11.0
+
+* Show cumulative CPU time in the job:list and job:show outputs
+* The error message when the specified schema has columns containing upper case
+  alphanumeric characters is improved to be more representative of the problem
+* 'td query' commands with result output to Treasure Data (--result td://xxxx)
+  validate the database and table naming convention before running the query
+* The Java bulk import JAR file is now auto-updated. Checking for an updated
+  version is performed hourly
+* 'td query' commands with result output specification invite the user to use
+  the '-x / --exclude' option to avoid outputting the query result to stdout as
+  well
+* The 'Destination' field in the summary for the Bulk import perform output
+  from the jobs:show command contains the destination table name in the form of
+  a LOAD DATA SESSION query
+* Returning the correct non-zero error codes in the occurrence of an exception
+* Declare the 'td query' option '--sampling' obsolete. A warning indicating the
+  option is obsolete and has no effect will be printed to warn the user
+
 == 2014-02-26 version 0.10.99
 
 * job:show, query: limit the number of records ouputted when printing on stdout using the -l / --limit option
 * job:show, query: optionally output/store csv and tsv files with headers using the -c / --column-names option
 * job:show, query: support result output from Presto
 
-
 == 2014-02-21 version 0.10.98
 
 * import:unfreeze: fixed NoMethodError error

data/Rakefile

@@ -11,7 +11,7 @@ require "erb"
 
 def version
   require project_root_path('lib/td/version')
-  TreasureData::VERSION
+  TreasureData::TOOLBELT_VERSION
 end
 
 task "jar" do

data/bin/td

@@ -1,17 +1,34 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
+
 require 'rubygems' unless defined?(gem)
 gem 'td-client'
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
 
+# disable the updater for the td gem
 require 'td/updater'
 TreasureData::Updater.disable(<<EOS
-`td update` is only available from Treasure Data Toolbelt.
-Download and install from http://toolbelt.treasure-data.com
+
+`td update` is only available from the Treasure Data Toolbelt.
+You can download and install it from http://toolbelt.treasure-data.com.
+
+It appers you are running the `td` gem. To update the gem to the latest
+version, please run `gem update td`.
+
+Please note that if you install `td` with `bundler` in a Gemfile/Gemspec
+federated environment, you will need to upgrade the reference version for
+`td` in the Gemfile/Gemspec for the updated `td` version to be used after
+updating it.
+
 EOS
 )
 
+# start up the CLI
 require 'td/command/runner'
-TreasureData::Command::Runner.new.run ARGV
-
+ev = TreasureData::Command::Runner.new.run ARGV
+unless ev.nil?
+  exit ev
+else
+  puts "No exit status"
+end

data/dist/resources/exe/td

@@ -27,4 +27,4 @@ TreasureData::Updater.inject_libpath
 
 # start up the CLI
 require 'td/command/runner'
-TreasureData::Command::Runner.new.run ARGV
+exit TreasureData::Command::Runner.new.run ARGV

data/dist/resources/pkg/td

@@ -26,5 +26,5 @@ require 'td/updater'
 TreasureData::Updater.inject_libpath
 
 # start up the CLI
-require "td/command/runner"
-TreasureData::Command::Runner.new.run ARGV
+require 'td/command/runner'
+exit TreasureData::Command::Runner.new.run ARGV

data/java/logging.properties

@@ -35,6 +35,7 @@ handlers= java.util.logging.FileHandler
 
 java.util.logging.FileHandler.level = INFO
 java.util.logging.FileHandler.pattern=td-bulk-import.log
+java.util.logging.FileHandler.append=true
 java.util.logging.FileHandler.limit = 50000
 java.util.logging.FileHandler.count = 1
 java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter

data/lib/td/command/common.rb

@@ -11,6 +11,12 @@ autoload :Job, 'td/client'
 
 module Command
 
+  class ParameterConfigurationError < ArgumentError
+  end
+
+  class BulkImportExecutionError < ArgumentError
+  end
+
   private
   def initialize
     @render_indent = ''
@@ -24,7 +30,7 @@ module Command
     unless apikey
       raise ConfigError, "Account is not configured."
     end
-    opts[:user_agent] = "TD: #{TreasureData::VERSION}"
+    opts[:user_agent] = "TD: #{TOOLBELT_VERSION}"
     if h = ENV['TD_API_HEADERS']
       pairs = h.split("\n")
       opts[:headers] = Hash[pairs.map {|pair| pair.split('=', 2) }]
@@ -105,30 +111,53 @@ EOS
     end
   end
 
-  def cmd_format_elapsed(start, finish)
+  def humanize_time(time, is_ms = false)
+    if time.nil?
+      return ''
+    end
+
+    time = time.to_i
+    millisecs = nil
+    elapsed = ''
+
+    if is_ms
+      # store the first 3 decimals
+      millisecs = time % 1000
+      time /= 1000
+    end
+
+    if time >= 3600
+      elapsed << "#{time / 3600}h "
+      time %= 3600
+      elapsed << "%dm " % (time / 60)
+      time %= 60
+      elapsed << "%ds" % time
+    elsif time >= 60
+      elapsed << "%dm " % (time / 60)
+      time %= 60
+      elapsed << "%ds" % time
+    elsif time > 0
+      elapsed << "%ds" % time
+    end
+
+    if is_ms and millisecs > 0
+      elapsed << " %03dms" % millisecs
+    end
+
+    elapsed
+  end
+
+  # assumed to
+  def humanize_elapsed_time(start, finish)
     if start
       if !finish
         finish = Time.now.utc
       end
-      e = finish.to_i - start.to_i
-      elapsed = ''
-      if e >= 3600
-        elapsed << "#{e/3600}h "
-        e %= 3600
-        elapsed << "%2dm " % (e/60)
-        e %= 60
-        elapsed << "%2dsec" % e
-      elsif e >= 60
-        elapsed << "%2dm " % (e/60)
-        e %= 60
-        elapsed << "%2dsec" % e
-      else
-        elapsed << "%2dsec" % e
-      end
+      elapsed = humanize_time(finish.to_i - start.to_i, false)
     else
       elapsed = ''
     end
-    elapsed = "% 13s" % elapsed  # right aligned
+    elapsed
   end
 
   def get_database(client, db_name)

data/lib/td/command/export.rb

@@ -9,7 +9,7 @@ module Command
     s3_bucket = nil
     aws_access_key_id = nil
     aws_secret_access_key = nil
-    file_format = nil
+    file_format = "json.gz" # default
 
     op.on('-f', '--from TIME', 'export data which is newer than or same with the TIME') {|s|
       from = export_parse_time(s)
@@ -26,7 +26,7 @@ module Command
     op.on('-s', '--aws-secret-key SECRET_KEY', 'AWS secret access key to export data (required)') {|s|
       aws_secret_access_key = s
     }
-    op.on('-F', '--file-format FILE_FORMAT', 'file format for exported data, either json.gz (default) or line-json.gz') { |s| 
+    op.on('-F', '--file-format FILE_FORMAT', 'file format for exported data, either json.gz (default) or line-json.gz') { |s|
       raise ArgumentError, "#{s} is not a supported file format" unless SUPPORTED_FORMATS.include?(s)
       file_format = s
     }

data/lib/td/command/import.rb

@@ -1,10 +1,9 @@
 require 'td/updater'
+require 'time'
 
 module TreasureData
 module Command
-
-  BASE_PATH = File.expand_path('../../..', File.dirname(__FILE__))
-  UPDATED_PATH = File.join(Updater.home_directory, ".td", "java")
+  include TreasureData::Updater
 
   JAVA_COMMAND = "java"
   JAVA_MAIN_CLASS = "com.treasure_data.td_import.BulkImportCommand"
@@ -25,37 +24,13 @@ module Command
     bulk_import_create(op)
   end
 
-  def import_java_version(op)
-    vfile = find_version_file[0]
-    puts "td-import-java #{File.open(vfile, 'r').read}"
+  def import_jar_version(op)
+    version = find_version_file
+    puts "td-import-java #{File.open(version, 'r').read}"
   end
 
   def import_jar_update(op)
-    last_updated = existence_jar_updated_time
-
-    require 'rexml/document'
-    require 'open-uri'
-    require 'fileutils'
-
-    doc = REXML::Document.new(open('http://maven.treasure-data.com/com/treasure_data/td-import/maven-metadata.xml') { |f| f.read })
-    updated = Time.strptime(REXML::XPath.match(doc, '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
-    version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
-
-    # Convert into UTF to compare time correctly
-    updated = (updated + updated.gmt_offset).utc unless updated.gmt?
-    last_updated = last_updated.utc unless last_updated.gmt?
-
-    if updated > last_updated
-      FileUtils.mkdir_p(UPDATED_PATH) unless File.exist?(UPDATED_PATH)
-      File.open(File.join(UPDATED_PATH, 'VERSION'), 'w') { |f| f.print "#{version} via import:jar_update" }
-      File.open(File.join(UPDATED_PATH, 'td-import-java.version'), 'w') { |f| f.print "#{version} #{updated}" }
-      File.open(File.join(UPDATED_PATH, 'td-import.jar'), 'wb') { |f|
-        f.print Updater.fetch("http://maven.treasure-data.com/com/treasure_data/td-import/#{version}/td-import-#{version}-jar-with-dependencies.jar")
-      }
-      puts "Installed td-import.jar #{version} into #{UPDATED_PATH}"
-    else
-      puts 'Installed td-import.jar is latest version'
-    end
+    check_n_update_jar(false)
   end
 
   def import_prepare(op)
@@ -100,8 +75,14 @@ module Command
     bulk_import_unfreeze(op)
   end
 
+  #
+  # Module private methods - don't map to import:* commands
+  #
+
   private
   def import_by_java(subcmd)
+    check_n_update_jar(true)
+
     # check java runtime exists or not
     check_java
 
@@ -122,6 +103,10 @@ module Command
 
     cmd = [JAVA_COMMAND] + JVM_OPTS + java_args
     system(*cmd)
+    if $?.exitstatus != 0
+      raise BulkImportExecutionError,
+            "Bulk Import returned error #{$?.exitstatus}. Please check the 'td-bulk-import.log' logfile for details."
+    end
   end
 
   private
@@ -136,20 +121,10 @@ module Command
 
     unless $?.success?
       $stderr.puts "Java is not installed. 'td import' command requires Java (version 1.6 or later)."
-      $stderr.puts "Alternatively, you can use 'bulk_import' commands instead which is much slower."
-      exit 1
-    end
-  end
-
-  private
-  def find_td_import_jar
-    libjars = find_files('*.jar')
-    found = libjars.find { |path| File.basename(path) =~ /^td-import/ }
-    if found.nil?
-      $stderr.puts "td-import.jar is not found."
+      $stderr.puts "Alternatively, you can use the 'bulk_import' commands."
+      $stderr.puts "Since they are implemented in Ruby, they perform significantly slower."
       exit 1
     end
-    found
   end
 
   private
@@ -158,13 +133,12 @@ module Command
 
     # set apiserver
     set_sysprops_endpoint(sysprops)
-
     # set http_proxy
     set_sysprops_http_proxy(sysprops)
 
     # set configuration file for logging
-    conf_file = try_find_logging_conf_file
-    if conf_file
+    conf_file = find_logging_property
+    unless conf_file.empty?
       sysprops << "-Djava.util.logging.config.file=#{conf_file}"
     end
 
@@ -223,30 +197,5 @@ module Command
     end
   end
 
-  private
-  def try_find_logging_conf_file
-    libjars = Dir.glob("#{BASE_PATH}/java/**/*.properties")
-    libjars.find { |path| File.basename(path) =~ /^logging.properties/ }
-  end
-
-  private
-  def find_version_file
-    vfile = find_files('VERSION')
-    vfile
-  end
-
-  def existence_jar_updated_time
-    require 'time'
-
-    content = File.open(find_files("td-import-java.version").first).read
-    index = content.index(' ')
-    Time.parse(content[index + 1..-1].strip)
-  end
-
-  def find_files(target)
-    files = Dir.glob("#{UPDATED_PATH}/**/#{target}")
-    return files unless files.empty?
-    Dir.glob("#{BASE_PATH}/java/**/#{target}")
-  end
 end
 end

data/lib/td/command/job.rb

@@ -67,17 +67,33 @@ module Command
       conditions = {:slower_than => slower_than}
     end
 
-    jobs = client.jobs(skip, skip+max-1, status, conditions)
+    jobs = client.jobs(skip, skip + max - 1, status, conditions)
 
     rows = []
     jobs.each {|job|
       start = job.start_at
-      elapsed = cmd_format_elapsed(start, job.end_at)
+      elapsed = humanize_elapsed_time(start, job.end_at)
+      cpu_time = humanize_time(job.cpu_time, true)
       priority = job_priority_name_of(job.priority)
-      rows << {:JobID => job.job_id, :Database => job.db_name, :Status => job.status, :Type => job.type, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Priority => priority, :Result => job.result_url}
+      rows << {
+        :JobID => job.job_id,
+        :Database => job.db_name,
+        :Status => job.status,
+        :Type => job.type,
+        :Query => job.query.to_s[0,50] + " ...",
+        :Start => (start ? start.localtime : ''),
+        :Elapsed => elapsed.rjust(11),
+        :CPUTime => cpu_time.rjust(17),
+        :Priority => priority,
+        :Result => job.result_url
+      }
     }
 
-    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Priority, :Result, :Type, :Database, :Query], :max_width => 140, :render_format => op.render_format)
+    puts cmd_render_table(rows,
+      :fields => [:JobID, :Status, :Start, :Elapsed, :CPUTime, :Priority, :Result, :Type, :Database, :Query],
+      :max_width => 1000,
+      :render_format => op.render_format
+    )
   end
 
   def job_show(op)
@@ -102,9 +118,9 @@ module Command
       output = s
       format = 'tsv' if format.nil?
     }
-    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
+    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json, msgpack, and msgpack.gz)') {|s|
       unless ['tsv', 'csv', 'json', 'msgpack', 'msgpack.gz'].include?(s)
-        raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack, msgpack.gz"
+        raise "Unknown format #{s.dump}. Supported formats are: tsv, csv, json, msgpack, and msgpack.gz"
       end
       format = s
     }
@@ -114,7 +130,8 @@ module Command
       end
       limit = s.to_i
     }
-    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to tsv and csv formats)', TrueClass) {|b|
+    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available',
+                                   '  for the table (only applies to tsv and csv formats)', TrueClass) {|b|
       render_opts[:header] = b;
     }
     op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
@@ -127,19 +144,21 @@ module Command
 
     if output.nil? && format
       unless ['tsv', 'csv', 'json'].include?(format)
-        raise "Supported formats are only tsv, csv and json without -o / --output option"
+        raise ParameterConfigurationError,
+              "Supported formats are only tsv, csv and json without -o / --output option"
       end
     end
 
     if render_opts[:header]
       unless ['tsv', 'csv'].include?(format)
-        raise "Option -c / --column-header is only supported with tsv and csv formats"
+        raise ParameterConfigurationError,
+              "Option -c / --column-header is only supported with tsv and csv formats"
       end
     end
 
     if !output.nil? && !limit.nil?
-      raise "Option -l / --limit is only valid when not outputting to file " +
-            "(no -o / --output option provided)"
+      raise ParameterConfigurationError,
+            "Option -l / --limit is only valid when not outputting to file (no -o / --output option provided)"
     end
 
     client = get_client
@@ -157,6 +176,12 @@ module Command
       puts "Retry limit : #{job.retry_limit}"
       puts "Output      : #{job.result_url}"
       puts "Query       : #{job.query}"
+    elsif job.type == :bulk_import_perform
+      puts "Destination : #{job.query}"
+    end
+    # if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
+    if job.finished? && [:hive].include?(job.type)
+      puts "CPU time    : #{humanize_time(job.cpu_time, true)}"
     end
 
     if wait && !job.finished?
@@ -183,14 +208,14 @@ module Command
       if verbose
         if !job.debug['cmdout'].nil?
           puts ""
-          puts "cmdout:"
+          puts "Output:"
           job.debug['cmdout'].to_s.split("\n").each {|line|
             puts "  " + line
           }
         end
         if !job.debug['stderr'].nil?
           puts ""
-          puts "stderr:"
+          puts "Details:"
           job.debug['stderr'].to_s.split("\n").each {|line|
             puts "  " + line
           }
@@ -198,7 +223,7 @@ module Command
       end
     end
 
-    $stderr.puts "Use '-v' option to show detailed messages." unless verbose
+    puts "Use '-v' option to show detailed messages." unless verbose
   end
 
   def job_status(op)
@@ -262,12 +287,33 @@ module Command
       write_result(job, output, limit, format, render_opts)
       puts "written to #{output} in #{format} format"
     else
+      # every format that is allowed on stdout
       render_result(job, limit, format, render_opts)
     end
   end
 
   def write_result(job, output, limit, format, render_opts={})
 
+    # start progress indicator
+    line_len = 0
+    start_time = last_time = Time.now.to_i
+    base_msg = "WARNING: the query result is being written"
+    if !output.nil?
+      msg = base_msg + "\r"
+      line_len += msg.length
+      print msg
+    end
+
+    # this lambda is passed on as a block when outputing the results to file
+    progress = lambda {
+      if (time = Time.now.to_i) - last_time > 5
+        msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
+        line_len = msg.length
+        print msg
+        last_time = time
+      end
+    }
+
     # the next 3 formats allow writing to both a file and stdout
 
     case format
@@ -281,6 +327,9 @@ module Command
           f.write Yajl.dump(row)
           n_rows += 1
           break if output.nil? and !limit.nil? and n_rows == limit
+
+          # update progress indicator
+          progress.call if !output.nil?
         }
         f.write "]"
       }
@@ -306,12 +355,17 @@ module Command
             dump_column(col)
           }
           n_rows += 1
+          writer.flush if n_rows % 100 == 0 # flush every 100 recods
           break if output.nil? and !limit.nil? and n_rows == limit
+
+          # update progress indicator
+          progress.call if !output.nil?
         }
       }
 
     when 'tsv'
       require 'yajl'
+
       open_file(output, "w") {|f|
         # output headers
         if render_opts[:header] && job.hive_result_schema
@@ -332,25 +386,43 @@ module Command
           }
           f.write "\n"
           n_rows += 1
+          f.flush if n_rows % 100 == 0 # flush every 100 recods
           break if output.nil? and !limit.nil? and n_rows == limit
+
+          # update progress indicator
+          progress.call if !output.nil?
         }
       }
 
     # these last 2 formats are only valid if writing the result to file through the -o/--output option.
 
     when 'msgpack'
+      if output.nil?
+        raise ParameterConfigurationError,
+              "Format 'msgpack' does not support writing to stdout"
+      end
       open_file(output, "wb") {|f|
-        job.result_format('msgpack', f)
+        job.result_format('msgpack', f, &progress)
       }
 
     when 'msgpack.gz'
+      if output.nil?
+        raise ParameterConfigurationError,
+              "Format 'msgpack' does not support writing to stdout"
+      end
       open_file(output, "wb") {|f|
-        job.result_format('msgpack.gz', f)
+        job.result_format('msgpack.gz', f, &progress)
       }
 
     else
       raise "Unknown format #{format.inspect}"
     end
+
+    # clear the progress indicator
+    if !output.nil?
+      print "\r"
+      print " " * (line_len + 5) + "\r"
+    end
   end
 
   def open_file(output, mode)
@@ -371,6 +443,14 @@ module Command
     require 'yajl'
 
     if format.nil?
+
+      # start progress indicator
+      start_time = last_time = Time.now.to_i
+      base_msg = "WARNING: the query result is being downloaded"
+      msg = base_msg + "\r"
+      line_len = msg.length
+      print msg
+
       # display result in tabular format
       rows = []
       n_rows = 0
@@ -381,8 +461,21 @@ module Command
         }
         n_rows += 1
         break if !limit.nil? and n_rows == limit
+
+        # progress indication
+        time = Time.now.to_i
+        if time - last_time > 5
+          msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
+          line_len = msg.length
+          print msg
+          last_time = time
+        end
       }
 
+      # clear the progress indicator
+      print "\r"
+      print " " * (line_len + 5) + "\r"
+
       render_opts[:max_width] = 10000
       if job.hive_result_schema
         render_opts[:change_fields] = job.hive_result_schema.map { |name,type| name }