td 0.11.1 → 0.11.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjhhYWI5NWE1ZmI3ZTkwZjY0OTNhMGE3N2VjNjUzNzk2ZWY2OGJkYQ==
+    M2EwMGZhM2U3ZmE4ZWE3YzE4ZGM4NTA3NDVmYzZlZTFmMmVkZDdiMg==
   data.tar.gz: !binary |-
-    NDZmMDJlMDQyZmViMjY5NTNmMTE0ODg5ZWE2MGYxZjJhYTg4Y2VhZQ==
+    MzUwZGIzYTVjOGE2Yzc1NDcyYWYzZTE2YjNlZjNkMzY3YTljYTY4OQ==
 SHA512:
   metadata.gz: !binary |-
-    NGZmMTk4ZTk5ZWFhYzc4YmEyZTE0ZmI0MjA2ODYzOWQxNzQ0ZTc5OGRjZDhh
-    OTI0NjJiZjA3MDhiZmFjNjg3MThjNjI1ZjcyZjNiOTNiOWU5MjQ2M2FhOWRh
-    ZTI4OGI2ZTk1YTEwYmVkZGJmNGU4Y2YzYTY1ZDc5MmNkYTI5ZTQ=
+    ZDE3YWM1OGQ5MWZmNTIxM2Q4NDNhMDdlMDUwYTcxNzYwZWUyNGU4YjRhNWFk
+    ZjZjOTFjNWRjZmQ0YWVhNmFlYTQyZDIyMTQ1NTBjNTA0NjI5YzlhMTMwMGE0
+    MDRlNDFiZDZiMTE1YjBlZmZkM2RkMzIxNDQ3NDc2OGEzNGMwY2M=
   data.tar.gz: !binary |-
-    NjNiNTI4MGI4MDY0MGYzMjZiMTBmNzBhYzJmNjNmZjcxZDgxZWVjNTgzYzUx
-    ZGZiMzViYmQ0ZjQwNWIwZmI5MWQ5ZmExYTA3MzZkZDlkY2RmNTEzNDMxYzNj
-    ZGJhZTFlZDExYjc4YzdkYzUwZmJjNjc3NTlkZGQ5OTI2MDQ4NjY=
+    OTY4NWJkYmJiNmU5NTRjYTUyZTA0MDg5ODQ1ZmM0NmMwMjY2MmUzMjhiZTdk
+    ZGE5MWQ1OWI4MzQ0YTlkMzcyY2YyMWVhNTc1ZjhhMTVkZTAyOTExZWU1OWM1
+    M2QzNjM5MTQ1ZmY0NzA0NTNjNTI4ZjQwNWIxNmMzYzY3YWY4YWM=

data/ChangeLog

@@ -1,3 +1,32 @@
+== 2014-06-18 version 0.11.2
+
+* Improve the update experience by showing a download progress indicator
+* Fix issue on Windows OS where the JAR version file handle was not closed
+  and caused permissions access problems to the file in subsequent attempts
+  to open the file
+* Add support for other/alternative backends using the -e / --endpoint
+  option. The backend can be also saved on the configuration file of use
+  via the 'td server:endpoint' or when running 'td -e <newendpoint> account'
+  command. The TD_API_SERVER environment variable still takes precendence
+  over all these settings
+* Removed Treasure Data Result Output database and table name client side
+  validation since it's performed in the API backend
+* Suppress stack traces for APIError exceptions from the client library
+* Changed the default HTTP port from 80 to 443 when interacting with the API
+  server to support SSL
+* Support working entirely without a configuration file (hence without
+  having to run 'td account' prior to using the CLI) by passing the -k option
+  on the command line
+* don't wipe out the current JAR file if an attempt to retrieve the most updated
+  JAR file failed because of network connectivity problems
+* Show the job result only for finished jobs: unfinished jobs don't have any
+  result to display anyway)
+* Honor the HTTP_PROXY setting also when updating the JAR file or the Toolbelt
+  itself
+* Fix parsing of the HTTP_PROXY variable for the Bulk Loader's usage
+* Allow optionally disabling the JAR file auto-update feature by setting the
+  TD_TOOLBELT_JAR_UPDATE environment variable to 0
+
 == 2014-04-29 version 0.11.1
 
 * Fix Treasure Data query result output database and table validation

data/README.rdoc

@@ -57,6 +57,70 @@ Then run following commands on MinGW Shell:
     $ bundle install  # don't use "--path" option
     $ rake exe:build  # don't use "bundle exec"
 
+== Testing Hooks
+
+The CLI implements several hooks to enable/disable/trigger special behaviors.
+These hooks are expressed as environment variables and can therefore be provided in several ways:
+
+=== How to Use
+
+* Unix / Linux / MacOSX
+  * environment variable export in the shell the command is executed. The setting remains active until the shell is closed. E.g.:
+  
+        $ export TD_TOOLBELT_DEBUG=1
+  
+  * in the shell configuration file, to be active in any new shell that is opened. E.g.: add
+  
+        export TD_TOOLBELT_DEBUG=1
+  
+    to <tt>~/.bashrc</tt> or equivalent shell configuration file.
+    To make the setting active in the current shell, source the configuration file, e.g.:
+    
+        $ source ~/.bashrc
+        
+  * on the command line at runtime (active only for the duration of the command). E.g.:
+        
+        $ TD_TOOLBELT_DEBUG=1 td ....
+        
+  * as alias on in the current shell. The setting remains active until the shell is closed. E.g.:
+        
+        $ alias td='TD_TOOLBELT_DEBUG=1 td'
+        
+  * as alias in configuration file, to be active in any new shell that is opened. E.g.:
+        
+        alias td='TD_TOOLBELT_DEBUG=1 td'`
+        
+    to <tt>~/.bashrc</tt> or equivalent shell configuration file.
+    To make the setting active in the current shell, source the configuration file, e.g.:
+    
+        $ source ~/.bashrc
+        
+* Windows
+  * in the command prompt the command is executed. The setting remains active until the command prompt window is closed. E.g.:
+    
+        cmd> set TD_TOOLBELT_DEBUG=1
+          
+  * as a global environment variable in the system settings. It will be active for all new command prompt windows.
+
+These are the available hooks:
+
+* Enable debugging mode:
+  
+    $ TD_TOOLBELT_DEBUG=1
+    
+* JAR auto update (enabled by default is not specified). This setting does not affect <tt>import:jar_update</tt>:
+  * Enable:
+        
+      $ TD_TOOLBELT_JAR_UPDATE=1
+        
+  * Disable:
+      
+      $ TD_TOOLBELT_JAR_UPDATE=0
+        
+* Specify an alternative endpoint to use to update the toolbelt (default: http://toolbelt.treasuredata.com): 
+  
+    $ TD_TOOLBELT_UPDATE_ROOT="http://toolbelt.treasuredata.com"
+    
 
 = Copyright
 

data/Rakefile

@@ -42,6 +42,17 @@ def install_use_gems(target_dir)
   ENV['GEM_PATH'] = ''
   USE_GEMS.each {|gem|
     begin
+      # this is a hack to have the dependency handling for the 'td' gem
+      #   pick up a local gem for 'td-client' so as to be able to build
+      #   and test the 'toolbelt' package without publishing the 'td-client'
+      #   gem on rubygems.com
+      unless ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM'].nil?
+        unless File.exists? ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']
+          raise "Cannot find gem file with path #{ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']}"
+        end
+        puts "Copy local gem #{ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']} to #{Dir.pwd}"
+        FileUtils.cp File.expand_path(ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']), Dir.pwd
+      end
       Gem::GemRunner.new.run ["install", gem, "--no-rdoc", "--no-ri"]
     rescue Gem::SystemExitException => e
       unless e.exit_code.zero?

data/dist/exe.rake

@@ -4,7 +4,7 @@ task 'exe:build' => :build do
   create_build_dir('exe') do |dir|
     # create ./installers/
     FileUtils.mkdir_p "installers"
-    installer_path = download_resource('http://rubyforge.org/frs/download.php/76054/rubyinstaller-1.9.3-p194.exe')
+    installer_path = download_resource('http://dl.bintray.com/oneclick/rubyinstaller/rubyinstaller-1.9.3-p545.exe?direct')
     FileUtils.cp installer_path, "installers/rubyinstaller.exe"
 
     variables = {

data/dist/pkg.rake

@@ -18,10 +18,10 @@ task 'pkg:build' => :build do
 
     zip_files(project_root_path('pkg/td-update-pkg.zip'), 'td-client.build')
 
-    # crete ./bundle/td-client.pkg/Bom
+    # create ./bundle/td-client.pkg/Bom
     sh "mkbom -s td-client.build bundle/td-client.pkg/Bom"
 
-    # crete ./bundle/td-client.pkg/Scripts/
+    # create ./bundle/td-client.pkg/Scripts/
     install_resource 'pkg/postinstall', 'bundle/td-client.pkg/Scripts/postinstall', 0755
 
     variables = {

data/lib/td/command/account.rb

@@ -13,11 +13,22 @@ module Command
 
     user_name = op.cmd_parse
 
+    endpoint = nil
+    # user may be calling 'td account' with the -e / --endpoint
+    # option, which we want to preserve and save
+    begin
+       endpoint = Config.endpoint
+    rescue ConfigNotFoundError => e
+      # the endpoint is neither stored in the config file
+      # nor passed as option on the command line
+    end
+
     conf = nil
     begin
       conf = Config.read
     rescue ConfigError
     end
+
     if conf && conf['account.apikey']
       unless force
         if conf['account.user']
@@ -68,7 +79,10 @@ module Command
 
       begin
         # enalbe SSL for the authentication
-        client = Client.authenticate(user_name, password, :ssl=>true)
+        opts = {}
+        opts[:ssl] = true
+        opts[:endpoint] = endpoint if endpoint
+        client = Client.authenticate(user_name, password, opts)
       rescue TreasureData::AuthError
         $stderr.puts "User name or password mismatched."
       end
@@ -77,28 +91,27 @@ module Command
     end
     return unless client
 
-    $stderr.puts "Authenticated successfully."
+    puts "Authenticated successfully."
 
     conf ||= Config.new
     conf["account.user"] = user_name
     conf["account.apikey"] = client.apikey
+    conf['account.endpoint'] = endpoint if endpoint
     conf.save
 
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db_name>' to create a database."
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create <db_name>' to create a database."
   end
 
   def account_usage(op)
     op.cmd_parse
 
     client = get_client
-
     a = client.account
 
     $stderr.puts "Storage:  #{a.storage_size_string}"
   end
 
   private
-
   if Helpers.on_windows?
     require 'Win32API'
 

data/lib/td/command/acl.rb

@@ -18,7 +18,7 @@ module Command
 
     if rows.empty?
       $stderr.puts "There are no access controls."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "acl:grant <subject> <action> <scope>' to grant permissions."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "acl:grant <subject> <action> <scope>' to grant permissions."
     end
   end
 

data/lib/td/command/apikey.rb

@@ -55,9 +55,8 @@ module Command
     conf["account.apikey"] = apikey
     conf.save
 
-    $stderr.puts "API key is set."
-
-    $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
+    puts "API key is set."
+    puts "Use '#{$prog} db:create <db_name>' to create a database."
   end
 
 end

data/lib/td/command/bulk_import.rb

@@ -20,7 +20,7 @@ module Command
 
     if rows.empty?
       $stderr.puts "There are no bulk import sessions."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:create <name> <db> <table>' to create a session."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "bulk_import:create <name> <db> <table>' to create a session."
     end
   end
 
@@ -61,7 +61,7 @@ module Command
     bi = client.bulk_import(name)
     unless bi
       $stderr.puts "Bulk import session '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:create <name> <db> <table>' to create a session."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "bulk_import:create <name> <db> <table>' to create a session."
       exit 1
     end
 
@@ -184,7 +184,7 @@ module Command
       job = client.perform_bulk_import(name)
 
       $stderr.puts "Job #{job.job_id} is queued."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show [-w] #{job.job_id}' to show the status."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show [-w] #{job.job_id}' to show the status."
     end
   end
 
@@ -246,7 +246,7 @@ module Command
         if bi.status == 'performing'
           $stderr.puts "Bulk import session '#{name}' is already performing."
           $stderr.puts "Add '-f' option to force start."
-          $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:kill #{bi.job_id}' to cancel the last trial."
+          $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:kill #{bi.job_id}' to cancel the last trial."
           exit 1
         elsif bi.status == 'ready'
           $stderr.puts "Bulk import session '#{name}' is already ready to commit."
@@ -259,7 +259,7 @@ module Command
     job = client.perform_bulk_import(name)
 
     $stderr.puts "Job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show [-w] #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show [-w] #{job.job_id}' to show the status."
 
     if wait
       require 'td/command/job'  # wait_job
@@ -330,13 +330,13 @@ module Command
     bi = bis.find {|bi| name == bi.name }
     unless bi
       $stderr.puts "Bulk import session '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:create <name> <db> <table>' to create a session."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "bulk_import:create <name> <db> <table>' to create a session."
       exit 1
     end
 
     if bi.status == "uploading" || bi.status == "performing"
       $stderr.puts "Bulk import session '#{name}' is not performed."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:perform <name>' to run."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "bulk_import:perform <name>' to run."
       exit 1
     end
 

data/lib/td/command/common.rb

@@ -17,6 +17,9 @@ module Command
   class BulkImportExecutionError < ArgumentError
   end
 
+  class UpdateError < ArgumentError
+  end
+
   private
   def initialize
     @render_indent = ''
@@ -26,15 +29,30 @@ module Command
     unless opts.has_key?(:ssl)
       opts[:ssl] = Config.secure
     end
+
+    # apikey is mandatory
     apikey = Config.apikey
-    unless apikey
-      raise ConfigError, "Account is not configured."
+    raise ConfigError, "Account is not configured." unless apikey
+
+    # optional, if not provided a default is used from the ruby client library
+    begin
+      if Config.endpoint
+        opts[:endpoint] = Config.endpoint
+      end
+    rescue ConfigNotFoundError => e
+      # rescue the ConfigNotFoundError exception which originates when
+      #   the config file is not found because the check on the apikey
+      #   guarantees that the API key has been provided on the command
+      #   line and that's good enough to continue since the default
+      #   endpoint will be used in place of this definition.
     end
+
     opts[:user_agent] = "TD: #{TOOLBELT_VERSION}"
     if h = ENV['TD_API_HEADERS']
       pairs = h.split("\n")
       opts[:headers] = Hash[pairs.map {|pair| pair.split('=', 2) }]
     end
+
     Client.new(apikey, opts)
   end
 
@@ -111,7 +129,7 @@ EOS
     end
   end
 
-  def humanize_time(time, is_ms = false)
+  def self.humanize_time(time, is_ms = false)
     if time.nil?
       return ''
     end
@@ -148,7 +166,7 @@ EOS
   end
 
   # assumed to
-  def humanize_elapsed_time(start, finish)
+  def self.humanize_elapsed_time(start, finish)
     if start
       if !finish
         finish = Time.now.utc
@@ -211,5 +229,72 @@ EOS
     end
   end
 
-end
-end
+  def self.validate_api_endpoint(endpoint)
+    require 'uri'
+
+    uri = URI.parse(endpoint)
+    unless uri.kind_of?(URI::HTTP) || uri.kind_of?(URI::HTTPS)
+      raise ParameterConfigurationError,
+            "API server endpoint URL must use 'http' or 'https' protocol. Example format: 'https://api.treasuredata.com'"
+    end
+
+    if !(md = /(\d{1,3}).(\d{1,3}).(\d{1,3}).(\d{1,3})/.match(uri.host)).nil? # IP address
+      md[1..-1].each { |v|
+        if v.to_i < 0 || v.to_i > 255
+          raise ParameterConfigurationError,
+                "API server IP address must a 4 integers tuple, with every integer in the [0,255] range. Example format: 'https://1.2.3.4'"
+        end
+      }
+    else # host name validation
+      unless uri.host =~ /\.treasure\-?data\.com$/
+        raise ParameterConfigurationError,
+              "API server endpoint URL must end with '.treasuredata.com' or '.treasure-data.com'. Example format: 'https://api.treasuredata.com'"
+      end
+      unless uri.host =~ /[\d\w\.]+\.treasure\-?data\.com$/
+        raise ParameterConfigurationError,
+              "API server endpoint URL must have prefix before '.treasuredata.com' or '.treasure-data.com'. Example format: 'https://api.treasuredata.com'."
+      end
+    end
+  end
+
+  def self.get_http_class
+    # use Net::HTTP::Proxy in place of Net::HTTP if a proxy is provided
+    http_proxy = ENV['HTTP_PROXY']
+    if http_proxy
+      http_proxy = (http_proxy =~ /\Ahttp:\/\/(.*)\z/) ? $~[1] : http_proxy
+      host, port = http_proxy.split(':', 2)
+      port = (port ? port.to_i : 80)
+      return Net::HTTP::Proxy(host, port)
+    else
+      return Net::HTTP
+    end
+  end
+
+  class DownloadProgressIndicator
+    def initialize(msg, start_time, periodicity)
+      @base_msg = msg
+      @start_time = start_time
+      @last_time = start_time
+      @periodicity = periodicity
+
+      print @base_msg + " " * 10
+    end
+  end
+
+  class TimeBasedDownloadProgressIndicator < DownloadProgressIndicator
+    def update
+      # progress indicator
+      if (time = Time.now.to_i) - @last_time > @periodicity
+        msg = "\r#{@base_msg}: #{Command.humanize_elapsed_time(@start_time, time)} elapsed"
+        print msg + " " * 10
+        @last_time = time
+      end
+    end
+
+    def finish
+      puts "\r#{@base_msg}...done" + " " * 20
+    end
+  end
+
+end # module Command
+end # module TrasureData