td 0.11.2 → 0.11.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2EwMGZhM2U3ZmE4ZWE3YzE4ZGM4NTA3NDVmYzZlZTFmMmVkZDdiMg==
+    MTZmYjMxNzU1OTk3MzQ0MzcyMGU5NzRhNzdkZjZhM2Q4ZTFlNDEyMw==
   data.tar.gz: !binary |-
-    MzUwZGIzYTVjOGE2Yzc1NDcyYWYzZTE2YjNlZjNkMzY3YTljYTY4OQ==
+    NzFhNTY5YjIyNmRlYTUzNzk1YWIxY2Q1MzMxMzc1YjFhODkxZmY4ZA==
 SHA512:
   metadata.gz: !binary |-
-    ZDE3YWM1OGQ5MWZmNTIxM2Q4NDNhMDdlMDUwYTcxNzYwZWUyNGU4YjRhNWFk
-    ZjZjOTFjNWRjZmQ0YWVhNmFlYTQyZDIyMTQ1NTBjNTA0NjI5YzlhMTMwMGE0
-    MDRlNDFiZDZiMTE1YjBlZmZkM2RkMzIxNDQ3NDc2OGEzNGMwY2M=
+    MTZkZGQyMWFkYjcxMDE2NWI2ZWEwZTVjNDQ5ODQ4YzYxMjMwOTJmZTI1Njc4
+    MGI4YWU1NzgzOTRmNjQ3ZDE5YjUxMTY2ZjhlMDM5Y2MxMmFjMWE4ZTEwNWM3
+    ZmZlYzM5ODRmYTQ0MTE0ZmRkMmFmNTNmNGYyMDU5MDhjNzRhOGE=
   data.tar.gz: !binary |-
-    OTY4NWJkYmJiNmU5NTRjYTUyZTA0MDg5ODQ1ZmM0NmMwMjY2MmUzMjhiZTdk
-    ZGE5MWQ1OWI4MzQ0YTlkMzcyY2YyMWVhNTc1ZjhhMTVkZTAyOTExZWU1OWM1
-    M2QzNjM5MTQ1ZmY0NzA0NTNjNTI4ZjQwNWIxNmMzYzY3YWY4YWM=
+    OWI1OGZjYTE4ODBjZDc4YWFiMDJhZWNmYmVlNjg3OTdkODA4Y2FlMjkyMDY5
+    OGY3M2M1N2QzYTMwMGFiZDhhM2RhZTQ3ODVkMjgxNWE0MGYzZjFhM2Y3Y2Ux
+    OGZiYTU0ZTFiOWEyZGYzZTgyMmRmMTFhOTY0Mzc2ZDM4Y2NlN2Q=

data/ChangeLog

@@ -1,3 +1,33 @@
+== 2014-07-31 version 0.11.3
+
+* Support name updates for schedules using sched:update command
+* Update Bulk Import JAR file repository to Central Maven Repository where
+  the Bulk Import JAR file is now hosted
+* Require Ruby v1.9 or superior for td to work ==> deprecated Ruby v1.8
+* Limit the number of keys in the input file (JSON, Msgoack) that can be
+  imported in one import to 512 as per the backend's limitations
+* Fixed import:create to work with import-only permissions and write-only API
+  keys
+* Handle NotFoundError, AuthError, and ForbiddenError to suppress printing of
+  the exception stack traces but yet show the exception class name for reference
+  back to the actual HTTP error code
+* validate the endpoints provided by -e / --endpoint using the Health-check
+  REST API /v3/system/server_status. Validation is avoided for any command
+  that does not require connectivity with the Treasure Data REST API server
+* JAR update errors are ignored when running import:prepare. This command
+  does not require connectivity and can be ran even when the machine is not
+  connected to the internet
+* Fixed Time based progress indicator to reflect the update interval exactly
+* Read logging.properties Logging configuration file from ~/.td/java first,
+  then the td repository's location (default). This allows customization of
+  the logging for the Java Bulk Import CLI (td-import-java).
+* Improve job result downloading by showing the total downloaded size and
+  percentage progress (Size based progress indicator). This is done when
+  printing the result in tabular format or writing the result on file in csv,
+  tsv, json, msgpack, or msgpack.gz formats. When displaying the result in csv,
+  tsv, and json format on stdout, the progress indicator is not used since the
+  data is progressively streamed in as it's received from the API server
+
 == 2014-06-18 version 0.11.2
 
 * Improve the update experience by showing a download progress indicator
@@ -7,7 +37,7 @@
 * Add support for other/alternative backends using the -e / --endpoint
   option. The backend can be also saved on the configuration file of use
   via the 'td server:endpoint' or when running 'td -e <newendpoint> account'
-  command. The TD_API_SERVER environment variable still takes precendence
+  command. The TD_API_SERVER environment variable still takes precedence
   over all these settings
 * Removed Treasure Data Result Output database and table name client side
   validation since it's performed in the API backend
@@ -82,7 +112,7 @@
 * Add --format option to some subcommands to specify render format. json, csv and tsv.
 * job:show and query subcommand now accept --format option without --output
 * Show -k option in command result message if -k option is given.
-* Support import:jar_update subcommand to update td-import seperately
+* Support import:jar_update subcommand to update td-import separately
 
 
 == 2013-11-20 version 0.10.95
@@ -109,10 +139,10 @@
 
 == 2013-11-05 version 0.10.92
 
-* Improve import commands. It can parse string values of 'HH:mm:ss' time format automatically without specifying '%T' and transfer into long unix timestamp.
-* Improve import commands. It can parse string values of 'yyyy/MM/dd HH:mm:ss' time format automatically without specifying 'time-format' and transfer into long unix timestamp.
-* Improve import commands. It can parse string values of 'yyyy/MM/dd HH:mm:ss Z' time format automatically without specifying 'time-format' and transfer into long unix timestamp.
-* Add 'regex' to source file format option of import:prepare command.
+* Improve import commands. It can parse string values of 'HH:mm:ss' time format automatically without specifying '%T' and transfer into long Unix timestamp.
+* Improve import commands. It can parse string values of 'yyyy/MM/dd HH:mm:ss' time format automatically without specifying 'time-format' and transfer into long Unix timestamp.
+* Improve import commands. It can parse string values of 'yyyy/MM/dd HH:mm:ss Z' time format automatically without specifying 'time-format' and transfer into long Unix timestamp.
+* Add 'regexp' to source file format option of import:prepare command.
 * Fix import:upload command. NullPointerException often occurs when data files are uploaded.
 
 
@@ -205,7 +235,7 @@
 == 2013-05-27 version 0.10.78
 
 * Add -q(--query) option to sched:create command
-* Add -x option to job:show, exlude result from automatically being downloaded
+* Add -x option to job:show, exclude result from automatically being downloaded
 * Update parallel gem dependency to 0.6.1 or later
 * user:add now requires email and password
 * td-client v0.8.50
@@ -214,8 +244,8 @@
 == 2013-05-06 version 0.10.77
 
 * Fix 'invalid byte sequence in UTF-8' at job result writing to CSV
-* Fix installed version check mechanizm of auto updater
-* Remove using curses to detect terminal height for Windows unexpected behaviour
+* Fix installed version check mechanism of auto updater
+* Remove using curses to detect terminal height for Windows unexpected behavior
 * Set User-Agent header
 
 
@@ -280,7 +310,8 @@
 
 == 2013-01-22 version 0.10.68
 
-* Fixed cmdline argument handling problems of bulk_import:prepare_parts2 subcommand
+* Fixed command-line argument handling problems of
+  bulk_import:prepare_parts2 subcommand
 
 
 == 2013-01-18 version 0.10.67
@@ -412,7 +443,7 @@ results on console
 
 * bulk_import:prepare_parts supports --true and --false type conversion options
 * bulk_import:prepare_parts supports --time-value option
-* bulk_import:prepare_parts convers /null/i to NULL by default
+* bulk_import:prepare_parts converts /null/i to NULL by default
 
 
 == 2012-09-04 version 0.10.44
@@ -432,7 +463,7 @@ results on console
 
 == 2012-08-30 version 0.10.41
 
-* Supports HTTP_PROXY environemnt variable to use a proxy server
+* Supports HTTP_PROXY environment variable to use a proxy server
 
 
 == 2012-08-07 version 0.10.40
@@ -662,7 +693,7 @@ results on console
 * Added new feature: status
 * Refined usage message
 * Fixed argument length check
-* help subcommand shows commandline examples
+* help subcommand shows command-line examples
 * table:tail subcommand reduced default max row number
 * job:list subcommand supports --running and --error options
 
@@ -759,7 +790,7 @@ results on console
 
 == 2011-09-08 version 0.9.0
 
-* Changed subcommands grammer: action-group -> group:action
+* Changed subcommands grammar: action-group -> group:action
 * Fixed import subcommand to strip \n
 
 

data/README.rdoc

@@ -1,9 +1,9 @@
 = Treasure Data command line tool
 
-This CUI utility wraps the td-client-ruby (https://github.com/treasure-data/td-client-ruby),
-the REST API for managing databases and jobs on the Treasure Data Cloud.
+This CUI utility wraps the {Ruby Client Library td-client-ruby}[https://github.com/treasure-data/td-client-ruby]
+to interact with the REST API in managing databases and jobs on the Treasure Data Cloud.
 
-For more about Treasure Data, see <http://treasure-data.com/>.
+For more info about Treasure Data, see <http://treasure-data.com/>.
 
 For full documentation see <http://docs.treasure-data.com/>.
 
@@ -57,6 +57,24 @@ Then run following commands on MinGW Shell:
     $ bundle install  # don't use "--path" option
     $ rake exe:build  # don't use "bundle exec"
 
+== Bulk Import
+
+Some of the +td+ commands prefixed with +td+ +import+ leverages the {Java Bulk Import CLI td-import-java}[https://github.com/treasure-data/td-import-java]
+to process and Bulk load data in the Treasure Data Cloud.
+
+The Bulk Import CLI is downloaded automatically at the first call to any of the command that require it; the use will
+need internet connectivity in order to fetch the Bulk Import CLI JAR file from the
+{Central Maven repository}[http://central.maven.org/maven2/com/treasuredata/td-import/]
+and take advantage of these advanced features. If you need to setup a proxy, please consult this
+{documentation}[http://docs.treasuredata.com/articles/faq#how-can-i-use-the-td-command-through-a-proxy] page.
+
+The log levels and properties of the Bulk Import CLI can be configured in a +logging.properties+ file. A default
+configuration is provided in a file within the gem or Toolbelt folder root, in the +java/+ folder. If you wish to
+customize it, please make a copy of this file and store it in the:
+
+    ~/.td/java             folder on Mac OSX or Linux
+    %USERPROFILE%\.td\java folder on Windows
+
 == Testing Hooks
 
 The CLI implements several hooks to enable/disable/trigger special behaviors.
@@ -66,61 +84,65 @@ These hooks are expressed as environment variables and can therefore be provided
 
 * Unix / Linux / MacOSX
   * environment variable export in the shell the command is executed. The setting remains active until the shell is closed. E.g.:
-  
+
         $ export TD_TOOLBELT_DEBUG=1
-  
+
   * in the shell configuration file, to be active in any new shell that is opened. E.g.: add
-  
+
         export TD_TOOLBELT_DEBUG=1
-  
+
     to <tt>~/.bashrc</tt> or equivalent shell configuration file.
     To make the setting active in the current shell, source the configuration file, e.g.:
-    
+
         $ source ~/.bashrc
-        
+
   * on the command line at runtime (active only for the duration of the command). E.g.:
-        
+
         $ TD_TOOLBELT_DEBUG=1 td ....
-        
+
   * as alias on in the current shell. The setting remains active until the shell is closed. E.g.:
-        
+
         $ alias td='TD_TOOLBELT_DEBUG=1 td'
-        
+
   * as alias in configuration file, to be active in any new shell that is opened. E.g.:
-        
+
         alias td='TD_TOOLBELT_DEBUG=1 td'`
-        
+
     to <tt>~/.bashrc</tt> or equivalent shell configuration file.
     To make the setting active in the current shell, source the configuration file, e.g.:
-    
+
         $ source ~/.bashrc
-        
+
 * Windows
   * in the command prompt the command is executed. The setting remains active until the command prompt window is closed. E.g.:
-    
+
         cmd> set TD_TOOLBELT_DEBUG=1
-          
+
   * as a global environment variable in the system settings. It will be active for all new command prompt windows.
 
 These are the available hooks:
 
 * Enable debugging mode:
-  
+
     $ TD_TOOLBELT_DEBUG=1
-    
+
 * JAR auto update (enabled by default is not specified). This setting does not affect <tt>import:jar_update</tt>:
   * Enable:
-        
+
       $ TD_TOOLBELT_JAR_UPDATE=1
-        
+
   * Disable:
-      
+
       $ TD_TOOLBELT_JAR_UPDATE=0
-        
-* Specify an alternative endpoint to use to update the toolbelt (default: http://toolbelt.treasuredata.com): 
-  
+
+* Specify an alternative endpoint to use updating the toolbelt (default: http://toolbelt.treasuredata.com):
+
     $ TD_TOOLBELT_UPDATE_ROOT="http://toolbelt.treasuredata.com"
-    
+
+* Specify an alternative endpoint to use updating the JAR file (default: http://central.maven.org):
+
+    $ TD_TOOLBELT_JARUPDATE_ROOT="http://central.maven.org"
+
 
 = Copyright
 

data/dist/exe.rake

@@ -23,7 +23,7 @@ task 'exe:build' => :build do
       install_resource 'exe/td-cmd.bat', 'td-cmd.bat', 0755
     end
 
-    zip_files(project_root_path('pkg/td-update-exe.zip'), 'td')
+    zip_files(project_root_path("pkg/td-update-exe-#{version}.zip"), 'td')
 
     # create td.iss and run Inno Setup
     install_erb_resource 'exe/td.iss', 'td.iss', 0644, variables
@@ -41,6 +41,6 @@ desc "clean Windows exe package"
 task "exe:clean" do
   FileUtils.rm_rf build_dir_path('exe')
   FileUtils.rm_rf project_root_path("pkg/td-#{version}.exe")
-  FileUtils.rm_rf project_root_path("pkg/td-update-exe.zip")
+  FileUtils.rm_rf project_root_path("pkg/td-update-exe-#{version}.zip")
 end
 

data/dist/pkg.rake

@@ -16,7 +16,7 @@ task 'pkg:build' => :build do
       sh "pax -wz -x cpio . > ../bundle/td-client.pkg/Payload"
     end
 
-    zip_files(project_root_path('pkg/td-update-pkg.zip'), 'td-client.build')
+    zip_files(project_root_path("pkg/td-update-pkg-#{version}.zip"), 'td-client.build')
 
     # create ./bundle/td-client.pkg/Bom
     sh "mkbom -s td-client.build bundle/td-client.pkg/Bom"
@@ -49,6 +49,6 @@ desc "clean Mac OS X pkg package"
 task "pkg:clean" do
   FileUtils.rm_rf build_dir_path('pkg')
   FileUtils.rm_rf project_root_path("pkg/td-#{version}.pkg")
-  FileUtils.rm_rf project_root_path("pkg/td-update-pkg.zip")
+  FileUtils.rm_rf project_root_path("pkg/td-update-pkg-#{version}.zip")
 end
 

data/lib/td/command/bulk_import.rb

@@ -28,11 +28,7 @@ module Command
     name, db_name, table_name = op.cmd_parse
 
     client = get_client
-
-    table = get_table(client, db_name, table_name)
-
-    opts = {}
-    client.create_bulk_import(name, db_name, table_name, opts)
+    client.create_bulk_import(name, db_name, table_name, {})
 
     $stderr.puts "Bulk import session '#{name}' is created."
   end

data/lib/td/command/common.rb

@@ -20,6 +20,9 @@ module Command
   class UpdateError < ArgumentError
   end
 
+  class ImportError < RuntimeError
+  end
+
   private
   def initialize
     @render_indent = ''
@@ -108,9 +111,9 @@ module Command
 
   def normalized_message
     <<EOS
-Your event has large number larger than 2^64.
+Your event has numbers larger than 2^64.
 These numbers are converted into string type.
-So you should use cast operator, e.g. cast(v['key'] as decimal), in your query.
+You should consider using the cast operator in your query: e.g. cast(v['key'] as decimal).
 EOS
   end
 
@@ -125,8 +128,42 @@ EOS
       $!.backtrace.each {|b|
         $stderr.puts "  #{b}"
       }
-        $stderr.puts ""
+      $stderr.puts ""
+    end
+  end
+
+  def self.humanize_bytesize(size, fractional_digits = 1)
+    labels = ["B", "kB", "MB", "GB", "TB"]
+    max_power = labels.length - 1
+
+    size = size.to_i
+
+    if size == 0
+      return "0 B"
+    end
+
+    # integer part
+    value = size
+    power = 0
+    while value > 0 do
+      value /= 1024
+      power += 1
     end
+    power -= 1
+    power = power > max_power ? max_power : power # limit to TB display
+    integer = size / (1024 ** power)
+    out = integer.to_s
+
+    # fractional part - remove the integer part to avoid running into floating
+    # point precision problems
+    value = size % ((1024 ** power) * integer)
+    if value > 0 and fractional_digits > 0
+      fractional = value.to_f / (1024 ** power)
+      fractional *= 10 ** fractional_digits
+      out += sprintf ".%0*d", fractional_digits, fractional.to_i
+    end
+    out += " " + labels[power.to_i]
+    out
   end
 
   def self.humanize_time(time, is_ms = false)
@@ -237,6 +274,10 @@ EOS
       raise ParameterConfigurationError,
             "API server endpoint URL must use 'http' or 'https' protocol. Example format: 'https://api.treasuredata.com'"
     end
+    unless uri.path =~ /^\/*$/
+      raise ParameterConfigurationError,
+            "API server path must be empty ('#{uri.path}' provided). Example format: 'https://api.treasuredata.com'"
+    end
 
     if !(md = /(\d{1,3}).(\d{1,3}).(\d{1,3}).(\d{1,3})/.match(uri.host)).nil? # IP address
       md[1..-1].each { |v|
@@ -245,15 +286,17 @@ EOS
                 "API server IP address must a 4 integers tuple, with every integer in the [0,255] range. Example format: 'https://1.2.3.4'"
         end
       }
-    else # host name validation
-      unless uri.host =~ /\.treasure\-?data\.com$/
-        raise ParameterConfigurationError,
-              "API server endpoint URL must end with '.treasuredata.com' or '.treasure-data.com'. Example format: 'https://api.treasuredata.com'"
-      end
-      unless uri.host =~ /[\d\w\.]+\.treasure\-?data\.com$/
-        raise ParameterConfigurationError,
-              "API server endpoint URL must have prefix before '.treasuredata.com' or '.treasure-data.com'. Example format: 'https://api.treasuredata.com'."
-      end
+    end
+  end
+
+  def self.test_api_endpoint(endpoint)
+    begin
+      # although the API may return 'Server is down' that is a good enough
+      #   indication that we hit a valid TD endpoint to accept it
+      Client.server_status(:endpoint => endpoint)
+    rescue
+      raise ParameterConfigurationError,
+            "The specified API server endpoint (#{endpoint}) does not respond."
     end
   end
 
@@ -270,24 +313,39 @@ EOS
     end
   end
 
+  def self.find_files(glob, locations)
+    files = []
+    locations.each {|loc|
+      files = Dir.glob("#{loc}/#{glob}")
+      break unless files.empty?
+    }
+    files
+  end
+
   class DownloadProgressIndicator
-    def initialize(msg, start_time, periodicity)
+    def initialize(msg)
       @base_msg = msg
-      @start_time = start_time
-      @last_time = start_time
-      @periodicity = periodicity
-
       print @base_msg + " " * 10
     end
   end
 
   class TimeBasedDownloadProgressIndicator < DownloadProgressIndicator
+    def initialize(msg, start_time, periodicity = 2)
+      @start_time = start_time
+      @last_time = start_time
+      @periodicity = periodicity
+      super(msg)
+    end
+
     def update
-      # progress indicator
-      if (time = Time.now.to_i) - @last_time > @periodicity
+      if (time = Time.now.to_i) - @last_time >= @periodicity
         msg = "\r#{@base_msg}: #{Command.humanize_elapsed_time(@start_time, time)} elapsed"
-        print msg + " " * 10
+        print "\r" + " " * (msg.length + 10)
+        print msg
         @last_time = time
+        true
+      else
+        false
       end
     end
 
@@ -296,5 +354,43 @@ EOS
     end
   end
 
+  class SizeBasedDownloadProgressIndicator < DownloadProgressIndicator
+    def initialize(msg, size, perc_step = 1, min_periodicity = nil)
+      @size = size
+
+      # perc_step is how small a percentage increment can be shown
+      @perc_step = perc_step
+
+      # min_periodicity=X limits updates to one every X second
+      @start_time = Time.now.to_i
+      @min_periodicity = min_periodicity
+
+      # track progress
+      @last_perc_step = 0
+      @last_time = @start_time
+
+      super(msg)
+    end
+
+    def update(curr_size)
+      ratio = (curr_size.to_f * 100 / @size).round(1)
+      if ratio >= (@last_perc_step + @perc_step) &&
+         (!@min_periodicity || (time = Time.now.to_i) - @last_time >= @min_periodicity)
+        msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)} / #{ratio}%"
+        print "\r" + " " * (msg.length + 10)
+        print msg
+        @last_perc_step = ratio
+        @last_time = time
+        true
+      else
+        false
+      end
+    end
+
+    def finish
+      print "\r#{@base_msg}...done" + " " * 20
+    end
+  end
+
 end # module Command
 end # module TrasureData

data/lib/td/command/help.rb

@@ -17,7 +17,7 @@ module Command
       exit 1
 
     else
-      method = List.get_method(cmd)
+      method, cmd_req_connectivity = List.get_method(cmd)
       method.call(['--help'])
     end
   end

data/lib/td/command/import.rb

@@ -34,15 +34,15 @@ module Command
   end
 
   def import_prepare(op)
-    import_by_java('prepare')
+    import_by_java(op)
   end
 
   def import_upload(op)
-    import_by_java('upload')
+    import_by_java(op)
   end
 
   def import_auto(op)
-    import_by_java('auto')
+    import_by_java(op)
   end
 
   def import_perform(op)
@@ -80,8 +80,17 @@ module Command
   #
 
   private
-  def import_by_java(subcmd)
-    check_n_update_jar(true)
+  def import_by_java(op)
+    subcmd = op.name.split(/:/)[1]
+    begin
+      check_n_update_jar(true)
+    rescue UpdateError => e
+      if op.cmd_requires_connectivity
+        raise e
+      else
+        puts "Warning: JAR update skipped for connectivity issues"
+      end
+    end
 
     # check java runtime exists or not
     check_java
@@ -227,5 +236,41 @@ module Command
     sysprops << "-Dhttp.proxyPort=#{proxy_port}" if proxy_port
   end
 
+  #
+  # Helpers
+  #
+
+  # find logging.properties file, first in the jarfile_dest_path, then in the
+  #   installed_path
+  def find_logging_property
+    installed_path = File.join(File.expand_path('../../..', File.dirname(__FILE__)), 'java')
+    config = Command.find_files("logging.properties", [Updater.jarfile_dest_path, installed_path])
+    if config.empty?
+      puts "Cannot find 'logging.properties' file in '#{Updater.jarfile_dest_path}' or " +
+           "'#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+      []
+    else
+      config.first
+    end
+  end
+
+  def find_td_import_jar
+    jar = Command.find_files('td-import.jar', [Updater.jarfile_dest_path])
+    if jar.empty?
+      $stderr.puts "Cannot find td-import.jar in '#{Updater.jarfile_dest_path}'."
+      exit 10
+    end
+    jar.first
+  end
+
+  def find_version_file
+    version = Command.find_files('VERSION', [Updater.jarfile_dest_path])
+    if version.empty?
+      $stderr.puts "Cannot find VERSION file in '#{Updater.jarfile_dest_path}'."
+      exit 10
+    end
+    version.first
+  end
+
 end
 end