td 0.11.1 → 0.11.2

data/lib/td/command/sample.rb

@@ -1,4 +1,3 @@
-
 module TreasureData
 module Command
 
@@ -23,9 +22,10 @@ module Command
       }
     }
 
-    $stderr.puts "Create #{fname} with #{i} records whose time is"
-    $stderr.puts "from #{Time.at(last_time)} to #{Time.at(t)}."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:import <db> <table> --json #{fname}' to import this file."
+    $stderr.print "Created #{fname} with #{i} records whose time is "
+    $stderr.puts "in the [#{Time.at(last_time)}, #{Time.at(t)}] range."
+
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:import <db> <table> --json #{fname}' to import this file."
   end
 
 end

data/lib/td/command/sched.rb

@@ -1,4 +1,3 @@
-
 module TreasureData
 module Command
 
@@ -127,7 +126,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -156,7 +155,7 @@ module Command
     op.on('-d', '--database DB_NAME', 'change the database') {|s|
       db_name = s
     }
-    op.on('-r', '--result RESULT_TABLE', 'change the result table') {|s|
+    op.on('-r', '--result RESULT_URL', 'change the result target (see also result:create subcommand)') {|s|
       result = s
     }
     op.on('-t', '--timezone TZ', "name of the timezone.",
@@ -210,7 +209,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -246,7 +245,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -302,7 +301,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -315,5 +314,5 @@ module Command
     puts cmd_render_table(rows, :fields => [:JobID, :Time], :max_width=>500, :render_format => op.render_format)
   end
 
-end
-end
+end # module Command
+end # module TreasureData

data/lib/td/command/server.rb

@@ -1,3 +1,4 @@
+require 'uri'
 
 module TreasureData
 module Command
@@ -8,6 +9,26 @@ module Command
     puts Client.server_status
   end
 
+  def server_endpoint(op)
+    endpoint = op.cmd_parse
+
+    Command.validate_api_endpoint(endpoint)
+
+    if Config.cl_endpoint and endpoint != Config.endpoint
+      raise ParameterConfigurationError,
+            "You specified the API server endpoint in the command options as well (-e / --endpoint option) but it does not match the value provided to the 'server:endpoint' command. Please remove the option or ensure the endpoints URLs match each other."
+    end
+
+    conf = nil
+    begin
+      conf = Config.read
+    rescue ConfigError
+      conf = Config.new
+    end
+    conf["account.endpoint"] = endpoint
+    conf.save
+  end
+
 end
 end
 

data/lib/td/command/status.rb

@@ -34,7 +34,7 @@ module Command
     j = client.jobs(0, 4)
     j.each {|job|
       start = job.start_at
-      elapsed = humanize_elapsed_time(start, job.end_at)
+      elapsed = Command.humanize_elapsed_time(start, job.end_at)
       jobs << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.result_url}
     }
     x2, y2 = status_render(0, 0, "[Jobs]", jobs, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query])
@@ -84,6 +84,6 @@ module Command
     return movex+max_width, height
   end
 
-end
-end
+end # module Command
+end # module TreasureData
 

data/lib/td/command/table.rb

@@ -40,7 +40,6 @@ module Command
 
     db_name, table_name = op.cmd_parse
 
-    #API.validate_database_name(db_name)
     API.validate_table_name(table_name)
 
     if HIVE_RESERVED_KEYWORDS.include?(table_name.upcase)
@@ -65,7 +64,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Database '#{db_name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create #{db_name}' to create the database."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create #{db_name}' to create the database."
       exit 1
     rescue AlreadyExistsError
       cmd_debug_error $!
@@ -111,7 +110,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Table '#{db_name}.#{table_name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:list #{db_name}' to show list of the tables."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:list #{db_name}' to show list of the tables."
       exit 1
     end
 
@@ -189,13 +188,13 @@ module Command
     if rows.empty?
       if db_name
         $stderr.puts "Database '#{db_name}' has no tables."
-        $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create <db> <table>' to create a table."
+        $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create <db> <table>' to create a table."
       elsif databases.empty?
         $stderr.puts "There are no databases."
-        $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db>' to create a database."
+        $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create <db>' to create a database."
       else
         $stderr.puts "There are no tables."
-        $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create <db> <table>' to create a table."
+        $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create <db> <table>' to create a table."
       end
     end
   end
@@ -336,7 +335,7 @@ module Command
     job = table.export('s3', opts)
 
     $stderr.puts "Export job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
 
     if wait && !job.finished?
       wait_job(job)
@@ -395,7 +394,7 @@ module Command
     job = client.partial_delete(db_name, table_name, to, from, opts)
 
     $stderr.puts "Partial delete job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
 
     if wait && !job.finished?
       wait_job(job)

data/lib/td/command/update.rb

@@ -14,12 +14,12 @@ module Command
     start_time = Time.now
     puts "Updating 'td' from #{TOOLBELT_VERSION}..."
     if new_version = Updater.update
-      puts "Successfully updated to #{new_version} in #{humanize_time((Time.now - start_time).to_i)}."
+      puts "Successfully updated to #{new_version} in #{Command.humanize_time((Time.now - start_time).to_i)}."
     else
       puts "Nothing to update."
     end
   end
 
-end
-end
+end # module Command
+end # module TreasureData
 

data/lib/td/command/user.rb

@@ -11,7 +11,7 @@ module Command
     user = users.find {|user| name == user.name }
     unless user
       $stderr.puts "User '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:create <name>' to create an user."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:create <name>' to create an user."
       exit 1
     end
 
@@ -37,7 +37,7 @@ module Command
 
     if rows.empty?
       $stderr.puts "There are no users."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:create <name>' to create an users."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:create <name>' to create an users."
     end
   end
 
@@ -120,7 +120,7 @@ module Command
     client.add_user(name, nil, email, password)
 
     $stderr.puts "User '#{name}' is created."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:apikeys #{name}' to show the API key."    
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:apikeys #{name}' to show the API key."    
   end
 
   def user_delete(op)
@@ -146,12 +146,12 @@ module Command
       client.add_apikey(name)
     rescue TreasureData::NotFoundError
       $stderr.puts "User '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "users' to show users."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "users' to show users."
       exit 1
     end
 
     $stderr.puts "Added an API key to user '#{name}'."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:apikeys #{name}' to show the API key"
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:apikeys #{name}' to show the API key"
   end
 
   def user_apikey_remove(op)
@@ -163,8 +163,8 @@ module Command
       client.remove_apikey(name, key)
     rescue TreasureData::NotFoundError
       $stderr.puts "User '#{name}' or API key '#{key}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "users' to show users."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:apikeys '#{key}' to show API keys"
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "users' to show users."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:apikeys '#{key}' to show API keys"
       exit 1
     end
 

data/lib/td/config.rb

@@ -13,6 +13,16 @@ end
 
 
 class Config
+  # class variables
+  @@path = ENV['TREASURE_DATA_CONFIG_PATH'] || ENV['TD_CONFIG_PATH'] || File.join(ENV['HOME'], '.td', 'td.conf')
+  @@apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
+  @@apikey = nil if @@apikey == ""
+  @@cl_apikey = false # flag to indicate whether an apikey has been provided through the command-line
+  @@endpoint = ENV['TREASURE_DATA_API_SERVER'] || ENV['TD_API_SERVER']
+  @@endpoint = nil if @@endpoint == ""
+  @@cl_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line
+  @@secure = true
+
   def initialize
     @path = nil
     @conf = {}   # section.key = val
@@ -76,23 +86,18 @@ class Config
     FileUtils.mkdir_p File.dirname(@path)
     File.open(@path, "w") {|f|
       @conf.keys.map {|cate_key|
-        cate_key.split('.',2)
-      }.zip(@conf.values).group_by {|(section,key),val|
+        cate_key.split('.', 2)
+      }.zip(@conf.values).group_by {|(section,key), val|
         section
       }.each {|section,cate_key_vals|
         f.puts "[#{section}]"
-        cate_key_vals.each {|(section,key),val|
+        cate_key_vals.each {|(section,key), val|
           f.puts "  #{key} = #{val}"
         }
       }
     }
   end
 
-  @@path = ENV['TREASURE_DATA_CONFIG_PATH'] || ENV['TD_CONFIG_PATH'] || File.join(ENV['HOME'], '.td', 'td.conf')
-  @@apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
-  @@apikey = nil if @@apikey == ""
-  @@cl_apikey = false # flag to indicate whether an apikey has been provided through the command-line
-  @@secure = true
 
   def self.path
     @@path
@@ -102,6 +107,16 @@ class Config
     @@path = path
   end
 
+
+  def self.secure
+    @@secure
+  end
+
+  def self.secure=(secure)
+    @@secure = secure
+  end
+
+
   def self.apikey
     @@apikey || Config.read['account.apikey']
   end
@@ -118,22 +133,32 @@ class Config
     @@cl_apikey = flag
   end
 
-  def self.cl_apikey_string
-    if @@cl_apikey
-      "-k #{@@apikey} "
-    else
-      ""
-    end
+
+  def self.endpoint
+    @@endpoint || Config.read['account.endpoint']
   end
 
-  def self.secure
-    @@secure
+  def self.endpoint=(endpoint)
+    @@endpoint = endpoint
   end
 
-  def self.secure=(secure)
-    @@secure = secure
+  def self.cl_endpoint
+    @@cl_endpoint
   end
-end
 
+  def self.cl_endpoint=(flag)
+    @@cl_endpoint = flag
+  end
 
-end
+  # renders the apikey and endpoint options as a string for the helper commands
+  def self.cl_options_string
+    string = ""
+    string += "-k #{@@apikey}" if @@cl_apikey
+    string += " " unless string.empty?
+    string += "-e #{@@endpoint}" if @@cl_endpoint
+    string += " " unless string.empty?
+    string
+  end
+
+end # class Config
+end # module TreasureData

data/lib/td/updater.rb

@@ -4,429 +4,442 @@ require "shellwords"
 require "zip/zip"
 
 module TreasureData
-  module Updater
-    #
-    # Toolbelt upgrade
-    #
-
-    def self.raise_error(message)
-      # TODO: Replace better Exception class
-      raise RuntimeError.new(message)
-    end
-
-    # copied from TreasureData::Helpers to avoid load issue.
-    def self.home_directory
-      on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME']
-    end
-
-    def self.on_windows?
-      RUBY_PLATFORM =~ /mswin32|mingw32/
-    end
-
-    def self.on_mac?
-      RUBY_PLATFORM =~ /-darwin\d/
-    end
-
-    def self.updating_lock_path
-      File.join(home_directory, ".td", "updating")
-    end
-
-    def self.installed_client_path
-      File.expand_path("../../../../../..", __FILE__)
-    end
-
-    def self.updated_client_path
-      File.join(home_directory, ".td", "updated")
-    end
-
-    def self.latest_local_version
-      installed_version = client_version_from_path(installed_client_path)
-      updated_version = client_version_from_path(updated_client_path)
-      if compare_versions(updated_version, installed_version) > 0
-        updated_version
-      else
-        installed_version
+module Updater
+  #
+  # Toolbelt upgrade
+  #
+
+  def self.raise_error(message)
+    # TODO: Replace better Exception class
+    raise RuntimeError.new(message)
+  end
+
+  # copied from TreasureData::Helpers to avoid load issue.
+  def self.home_directory
+    on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME']
+  end
+
+  def self.on_windows?
+    RUBY_PLATFORM =~ /mswin32|mingw32/
+  end
+
+  def self.on_mac?
+    RUBY_PLATFORM =~ /-darwin\d/
+  end
+
+  def self.updating_lock_path
+    File.join(home_directory, ".td", "updating")
+  end
+
+  def self.installed_client_path
+    File.expand_path("../../../../../..", __FILE__)
+  end
+
+  def self.updated_client_path
+    File.join(home_directory, ".td", "updated")
+  end
+
+  def self.latest_local_version
+    installed_version = client_version_from_path(installed_client_path)
+    updated_version = client_version_from_path(updated_client_path)
+    if compare_versions(updated_version, installed_version) > 0
+      updated_version
+    else
+      installed_version
+    end
+  end
+
+  def self.get_client_version_file(path)
+    td_gems = Dir[File.join(path, "vendor/gems/td-*")]
+    td_gems.each { |td_gem|
+      if td_gem =~ /#{"#{Regexp.escape(path)}\/vendor\/gems\/td-\\d*.\\d*.\\d*"}/
+        return File.join(td_gem, "/lib/td/version.rb")
       end
-    end
-
-    def self.get_client_version_file(path)
-      td_gems = Dir[File.join(path, "vendor/gems/td-*")]
-      td_gems.each { |td_gem|
-        if td_gem =~ /#{"#{Regexp.escape(path)}\/vendor\/gems\/td-\\d*.\\d*.\\d*"}/
-          return File.join(td_gem, "/lib/td/version.rb")
-        end
-      }
-      nil
-    end
-
-    def self.client_version_from_path(path)
-      if version_file = get_client_version_file(path)
-        File.read(version_file).match(/TOOLBELT_VERSION = '([^']+)'/)[1]
-      else
-        '0.0.0'
+    }
+    nil
+  end
+
+  def self.client_version_from_path(path)
+    if version_file = get_client_version_file(path)
+      File.read(version_file).match(/TOOLBELT_VERSION = '([^']+)'/)[1]
+    else
+      '0.0.0'
+    end
+  end
+
+  def self.disable(message)
+    @disable = message
+  end
+
+  def self.disable?
+    !@disable.nil?
+  end
+
+  def self.disable_message
+    @disable
+  end
+
+  def self.wait_for_lock(path, wait_for = 5, check_every = 0.5)
+    start = Time.now.to_i
+    while File.exists?(path)
+      sleep check_every
+      if (Time.now.to_i - start) > wait_for
+        raise_error "Unable to acquire update lock"
       end
     end
-
-    def self.disable(message)
-      @disable = message
-    end
-
-    def self.disable?
-      !@disable.nil?
-    end
-
-    def self.disable_message
-      @disable
-    end
-
-    def self.wait_for_lock(path, wait_for = 5, check_every = 0.5)
-      start = Time.now.to_i
-      while File.exists?(path)
-        sleep check_every
-        if (Time.now.to_i - start) > wait_for
-          raise_error "Unable to acquire update lock"
-        end
-      end
-      begin
-        FileUtils.touch(path)
-        ret = yield
-      ensure
-        FileUtils.rm_f(path)
-      end
-      ret
-    end
-
-    def self.package_category
-      case
-      when on_windows?
-        'exe'
-      when on_mac?
-        'pkg'
-      else
-        raise_error "Non supported environment"
-      end
-    end
-
-    def self.fetch(uri)
-      require 'net/http'
-      require 'openssl'
-
-      # open-uri can't treat 'http -> https' redirection and
-      # Net::HTTP.get_response can't get response from HTTPS endpoint.
-      # So we use following code to avoid above issues.
-      u = URI(uri)
-      response =
-        if u.scheme == 'https'
-          http = Net::HTTP.new(u.host, u.port)
-          http.use_ssl = true
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          http.request(Net::HTTP::Get.new(u.path))
-        else
-          Net::HTTP.get_response(u)
-        end
-
-      case response
-      when Net::HTTPSuccess then response.body
-      when Net::HTTPRedirection then fetch(response['Location'])
+    begin
+      FileUtils.touch(path)
+      ret = yield
+    ensure
+      FileUtils.rm_f(path)
+    end
+    ret
+  end
+
+  def self.package_category
+    case
+    when on_windows?
+      'exe'
+    when on_mac?
+      'pkg'
+    else
+      raise_error "Non supported environment"
+    end
+  end
+
+  def self.fetch(url)
+    require 'net/http'
+    require 'openssl'
+
+    http_class = Command.get_http_class
+
+    # open-uri can't treat 'http -> https' redirection and
+    # Net::HTTP.get_response can't get response from HTTPS endpoint.
+    # So we use following code to avoid these issues.
+    uri = URI(url)
+    response =
+      if uri.scheme == 'https' and ENV['HTTP_PROXY'].nil?
+        # NOTE: SSL is force off for communications over proxy
+        http = http_class.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        http.request(Net::HTTP::Get.new(uri.path))
       else
-        raise "An error occurred when fetching from '#{uri}'."
-        response.error!
+        http_class.get_response(uri)
       end
-    end
 
-    def self.endpoint_root
-      ENV['TD_TOOLBELT_UPDATE_ROOT'] || "http://toolbelt.treasuredata.com"
+    case response
+    when Net::HTTPSuccess then response.body
+    when Net::HTTPRedirection then fetch(response['Location'])
+    else
+      raise "An error occurred when fetching from '#{url}'."
+      response.error!
     end
+  end
 
-    def self.version_endpoint
-      "#{endpoint_root}/version.#{package_category}"
-    end
-
-    def self.update_package_endpoint
-      "#{endpoint_root}/td-update-#{package_category}.zip"
-    end
+  def self.endpoint_root
+    ENV['TD_TOOLBELT_UPDATE_ROOT'] || "http://toolbelt.treasuredata.com"
+  end
 
-    def self.update(autoupdate = false)
-      wait_for_lock(updating_lock_path, 5) do
-        require "td"
-        require 'open-uri'
-        require "tmpdir"
-        require "zip/zip"
-
-        latest_version = fetch(version_endpoint)
-
-        if compare_versions(latest_version, latest_local_version) > 0
-          Dir.mktmpdir do |download_dir|
-
-            # initialize the progress indicator
-            base_msg = "Downloading updated toolbelt package"
-            print base_msg + " " * 10
-            start_time = last_time = Time.new.to_i
-
-            # downloading the update compressed file
-            File.open("#{download_dir}/td-update.zip", "wb") do |file|
-              endpoint = update_package_endpoint
-              puts "\npackage '#{endpoint}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
-              stream_fetch(endpoint, file) {
-
-                # progress indicator
-                if (time = Time.now.to_i) - last_time > 2
-                  msg = "\r" + base_msg + ": #{time - start_time}s elapsed"
-                  # TODO parse 'time - start_time' with humanize_time from common.rb
-                  #      once this method is not static
-                  print msg + " " * 10
-                  last_time = time
-                end
-              }
-            end
+  def self.version_endpoint
+    "#{endpoint_root}/version.#{package_category}"
+  end
 
-            # clear progress indicator with this final message
-            puts "\r" + base_msg + "...done" + " " * 20
+  def self.update_package_endpoint
+    "#{endpoint_root}/td-update-#{package_category}.zip"
+  end
 
-            print "Unpacking updated toolbelt package..."
-            Zip::ZipFile.open("#{download_dir}/td-update.zip") do |zip|
-              zip.each do |entry|
-                target = File.join(download_dir, entry.to_s)
-                FileUtils.mkdir_p(File.dirname(target))
-                zip.extract(entry, target) { true }
-              end
+  def self.update(autoupdate = false)
+    wait_for_lock(updating_lock_path, 5) do
+      require "td"
+      require 'open-uri'
+      require "tmpdir"
+      require "zip/zip"
+
+      latest_version = fetch(version_endpoint)
+
+      if compare_versions(latest_version, latest_local_version) > 0
+        Dir.mktmpdir do |download_dir|
+
+          indicator = Command::TimeBasedDownloadProgressIndicator.new(
+            "Downloading updated toolbelt package", Time.new.to_i, 2)
+          # downloading the update compressed file
+          File.open("#{download_dir}/td-update.zip", "wb") do |file|
+            endpoint = update_package_endpoint
+            puts "\npackage '#{endpoint}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
+            stream_fetch(endpoint, file) {
+              indicator.update
+            }
+          end
+          indicator.finish
+
+          print "Unpacking updated toolbelt package..."
+          Zip::ZipFile.open("#{download_dir}/td-update.zip") do |zip|
+            zip.each do |entry|
+              target = File.join(download_dir, entry.to_s)
+              FileUtils.mkdir_p(File.dirname(target))
+              zip.extract(entry, target) { true }
             end
-            print "done\n"
+          end
+          print "done\n"
 
-            FileUtils.rm "#{download_dir}/td-update.zip"
+          FileUtils.rm "#{download_dir}/td-update.zip"
 
-            old_version = latest_local_version
-            new_version = client_version_from_path(download_dir)
+          old_version = latest_local_version
+          new_version = client_version_from_path(download_dir)
 
-            if compare_versions(new_version, old_version) < 0 && !autoupdate
-              raise_error "Installed version (#{old_version}) is newer than the latest available update (#{new_version})"
-            end
+          if compare_versions(new_version, old_version) < 0 && !autoupdate
+            raise_error "Installed version (#{old_version}) is newer than the latest available update (#{new_version})"
+          end
 
-            FileUtils.rm_rf updated_client_path
-            FileUtils.mkdir_p File.dirname(updated_client_path)
-            FileUtils.cp_r(download_dir, updated_client_path)
+          FileUtils.rm_rf updated_client_path
+          FileUtils.mkdir_p File.dirname(updated_client_path)
+          FileUtils.cp_r(download_dir, updated_client_path)
 
-            new_version
-          end
-        else
-          false # already up to date
+          new_version
         end
+      else
+        false # already up to date
       end
-    ensure
-      FileUtils.rm_f(updating_lock_path)
     end
+  ensure
+    FileUtils.rm_f(updating_lock_path)
+  end
 
-    def self.compare_versions(first_version, second_version)
-      first_version.split('.').map { |part| Integer(part) rescue part } <=> second_version.split('.').map { |part| Integer(part) rescue part }
-    end
+  def self.compare_versions(first_version, second_version)
+    first_version.split('.').map { |part| Integer(part) rescue part } <=> second_version.split('.').map { |part| Integer(part) rescue part }
+  end
 
-    def self.inject_libpath
-      old_version = client_version_from_path(installed_client_path)
-      new_version = client_version_from_path(updated_client_path)
+  def self.inject_libpath
+    old_version = client_version_from_path(installed_client_path)
+    new_version = client_version_from_path(updated_client_path)
 
-      if compare_versions(new_version, old_version) > 0
-        vendored_gems = Dir[File.join(updated_client_path, "vendor", "gems", "*")]
-        vendored_gems.each do |vendored_gem|
-          $:.unshift File.join(vendored_gem, "lib")
-        end
-        load('td/updater.rb') # reload updated updater
+    if compare_versions(new_version, old_version) > 0
+      vendored_gems = Dir[File.join(updated_client_path, "vendor", "gems", "*")]
+      vendored_gems.each do |vendored_gem|
+        $:.unshift File.join(vendored_gem, "lib")
       end
+      load('td/updater.rb') # reload updated updater
+    end
 
-      # check every hour if the toolbelt can be updated.
-      # => If so, update in the background
-      if File.exists?(last_toolbelt_autoupdate_timestamp)
-        return if (Time.now.to_i - File.mtime(last_toolbelt_autoupdate_timestamp).to_i) < 60 * 60 * 1 # every 1 hours
+    # check every hour if the toolbelt can be updated.
+    # => If so, update in the background
+    if File.exists?(last_toolbelt_autoupdate_timestamp)
+      return if (Time.now.to_i - File.mtime(last_toolbelt_autoupdate_timestamp).to_i) < 60 * 60 * 1 # every 1 hours
+    end
+    log_path = File.join(home_directory, '.td', 'autoupdate.log')
+    FileUtils.mkdir_p File.dirname(log_path)
+    td_binary = File.expand_path($0)
+    pid = if defined?(RUBY_VERSION) and RUBY_VERSION =~ /^1\.8\.\d+/
+      fork do
+        exec("#{Shellwords.escape(td_binary)} update &> #{Shellwords.escape(log_path)} 2>&1")
       end
-      log_path = File.join(home_directory, '.td', 'autoupdate.log')
-      FileUtils.mkdir_p File.dirname(log_path)
-      td_binary = File.expand_path($0)
-      pid = if defined?(RUBY_VERSION) and RUBY_VERSION =~ /^1\.8\.\d+/
-        fork do
-          exec("#{Shellwords.escape(td_binary)} update &> #{Shellwords.escape(log_path)} 2>&1")
+    else
+      log_file = File.open(log_path, "w")
+      spawn(td_binary, 'update', :err => log_file, :out => log_file)
+    end
+    Process.detach(pid)
+    FileUtils.mkdir_p File.dirname(last_toolbelt_autoupdate_timestamp)
+    FileUtils.touch last_toolbelt_autoupdate_timestamp
+  end
+
+  def self.last_toolbelt_autoupdate_timestamp
+    File.join(home_directory, ".td", "autoupdate.last")
+  end
+
+  #
+  # td-import.jar upgrade
+  #
+
+  # locate the root of the td package which is 3 folders up from the location of this file
+  def jarfile_dest_path
+    File.join(Updater.home_directory, ".td", "java")
+  end
+
+  private
+  def self.stream_fetch(url, binfile, &progress)
+    require 'net/http'
+
+    uri = URI(url)
+    http_class = Command.get_http_class
+    http = http_class.new(uri.host, uri.port)
+
+    # NOTE: keep SSL off when using a proxy
+    if uri.scheme == 'https' and ENV['HTTP_PROXY'].nil?
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+
+    http.request_get(uri.path) {|response|
+      if response.class == Net::HTTPOK
+        # print a . every tick_period seconds
+        response.read_body do |chunk|
+          binfile.write chunk
+          progress.call unless progress.nil?
         end
+        return true
+      elsif response.class == Net::HTTPFound || \
+            response.class == Net::HTTPRedirection
+        puts "redirect '#{url}' to '#{response['Location']}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
+        return stream_fetch(response['Location'], binfile, &progress)
       else
-        log_file = File.open(log_path, "w")
-        spawn(td_binary, 'update', :err => log_file, :out => log_file)
+        raise "An error occurred when fetching from '#{uri}' " +
+              "(#{response.class.to_s}: #{response.message})."
+        return false
       end
-      Process.detach(pid)
-      FileUtils.mkdir_p File.dirname(last_toolbelt_autoupdate_timestamp)
-      FileUtils.touch last_toolbelt_autoupdate_timestamp
-    end
+    }
+  end
 
-    def self.last_toolbelt_autoupdate_timestamp
-      File.join(home_directory, ".td", "autoupdate.last")
-    end
+  private
+  def jar_update(hourly = false)
+    require 'rexml/document'
+    require 'open-uri'
+    require 'fileutils'
 
-    #
-    # td-import.jar upgrade
-    #
+    maven_repo = "http://maven.treasure-data.com/com/treasure_data/td-import"
 
-    # locate the root of the td package which is 3 folders up from the location of this file
-    def jarfile_dest_path
-      File.join(Updater.home_directory, ".td", "java")
+    begin
+      xml = Updater.fetch("#{maven_repo}/maven-metadata.xml")
+    rescue Exception => exc
+      raise Command::UpdateError,
+            "There was a problem accessing the remote XML resource '#{maven_repo}/maven-metadata.xml' " +
+            "(#{exc.class.to_s}: #{exc.message})"
     end
-
-    private
-    def self.stream_fetch(url, binfile, &progress)
-      require 'net/http'
-
-      uri = URI(url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      if uri.scheme == 'https'
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
-      http.start
-      request = Net::HTTP::Get.new(uri.to_s)
-      http.request request do |response|
-        if response.class == Net::HTTPOK
-          # print a . every tick_period seconds
-          response.read_body do |chunk|
-            binfile.write chunk
-            progress.call #unless progress.nil?
-          end
-          return true
-        elsif response.class == Net::HTTPFound || \
-              response.class == Net::HTTPRedirection
-          puts "redirect '#{url}' to '#{response['Location']}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
-          return stream_fetch(response['Location'], binfile, &progress)
-        else
-          raise "An error occurred when fetching from '#{uri}'."
-          return false
-        end
-      end
+    if xml.nil? || xml.empty?
+      raise Command::UpdateError,
+            "The remote XML resource '#{maven_repo}/maven-metadata.xml' returned an empty file."
     end
 
-    private
-    def jar_update(hourly = false)
-      maven_repo = "http://maven.treasure-data.com/com/treasure_data/td-import"
+    # read version and update date from the xml file
+    doc = REXML::Document.new(xml)
+    updated = Time.strptime(REXML::XPath.match(doc, '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
+    version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
 
-      require 'rexml/document'
-      require 'open-uri'
-      require 'fileutils'
-
-      doc = REXML::Document.new(open("#{maven_repo}/maven-metadata.xml") { |f| f.read })
-      updated = Time.strptime(REXML::XPath.match(doc, '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
-      version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
-
-      # Convert into UTF to compare time correctly
-      updated = (updated + updated.gmt_offset).utc unless updated.gmt?
-      last_updated = existent_jar_updated_time
-
-      if updated > last_updated
-        FileUtils.mkdir_p(jarfile_dest_path) unless File.exists?(jarfile_dest_path)
-        Dir.chdir jarfile_dest_path
-
-        File.open('VERSION', 'w') { |f| f.print "#{version} via import:jar_update" }
-        File.open('td-import-java.version', 'w') { |f| f.print "#{version} #{updated}" }
-
-        # initialize the progress indicator
-        base_msg = "Updating td-import.jar"
-        start_time = last_time = Time.new.to_i
-        print base_msg + " " * 10
-
-        binfile = File.open 'td-import.jar.new', 'wb'
-        status = Updater.stream_fetch("#{maven_repo}/#{version}/td-import-#{version}-jar-with-dependencies.jar", binfile) {
-
-          # progress indicator
-          if (time = Time.now.to_i) - last_time > 2
-            msg = "\r" + base_msg + ": #{humanize_time(time - start_time)} elapsed"
-            # TODO parse 'time - start_time' with humanize_time from common.rb
-            #      once this method is not static
-            print msg + " " * 10
-            last_time = time
-          end
-        }
-        binfile.close
+    # Convert into UTF to compare time correctly
+    updated = (updated + updated.gmt_offset).utc unless updated.gmt?
+    last_updated = existent_jar_updated_time
 
-        # clear progress indicator with this final message
-        puts "\r" + base_msg + "...done" + " " * 20
+    if updated > last_updated
+      FileUtils.mkdir_p(jarfile_dest_path) unless File.exists?(jarfile_dest_path)
+      Dir.chdir jarfile_dest_path
 
-        if status
-          puts "Installed td-import.jar v#{version} in '#{jarfile_dest_path}'.\n"
-          File.rename 'td-import.jar.new', 'td-import.jar'
+      File.open('VERSION', 'w') {|f|
+        if hourly
+          f.print "#{version} via hourly jar auto-update"
         else
-          puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
-          File.delete 'td-import.jar.new' if File.exists? 'td-import.jar.new'
+          f.print "#{version} via import:jar_update command"
         end
-      else
-        puts 'Installed td-import.jar is already at the latest version.' unless hourly
-      end
-    end
-
-    def check_n_update_jar(hourly = false)
-      if hourly && \
-         File.exists?(last_jar_autoupdate_timestamp) && \
-         (Time.now - File.mtime(last_jar_autoupdate_timestamp)).to_i < (60 * 60 * 1) # every hour
-        return
-      end
-      jar_update(hourly)
-      FileUtils.touch File.join(jarfile_dest_path, "td-import-java.version")
-    end
-
-    private
-    def last_jar_autoupdate_timestamp
-      File.join(jarfile_dest_path, "td-import-java.version")
-    end
-
-    private
-    def existent_jar_updated_time
-      files = find_files("td-import-java.version", [jarfile_dest_path])
-      if files.empty?
-        return Time.at(0)
-      end
-      content = File.open(files.first).read
-      index = content.index(' ')
-      time = nil
-      if index.nil?
-        time = Time.at(0).utc
-      else
-        time = Time.parse(content[index+1..-1].strip).utc
-      end
-      time
-    end
+      }
+      File.open('td-import-java.version', 'w') {|f|
+        f.print "#{version} #{updated}"
+      }
 
-    #
-    # Helpers
-    #
-    def find_files(glob, locations)
-      files = []
-      locations.each {|loc|
-        files = Dir.glob("#{loc}/#{glob}")
-        break unless files.empty?
+      indicator = Command::TimeBasedDownloadProgressIndicator.new(
+        "Updating td-import.jar", Time.new.to_i, 2)
+      binfile = File.open 'td-import.jar.new', 'wb'
+      status = Updater.stream_fetch("#{maven_repo}/#{version}/td-import-#{version}-jar-with-dependencies.jar", binfile) {
+        indicator.update
       }
-      files
-    end
+      binfile.close
+      indicator.finish()
 
-    def find_version_file
-      version = find_files('VERSION', [jarfile_dest_path])
-      if version.empty?
-        $stderr.puts "Cannot find VERSION file in '#{jarfile_dest_path}'."
-        exit 10
+      if status
+        puts "Installed td-import.jar v#{version} in '#{jarfile_dest_path}'.\n"
+        File.rename 'td-import.jar.new', 'td-import.jar'
+      else
+        puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+        File.delete 'td-import.jar.new' if File.exists? 'td-import.jar.new'
       end
-      version.first
-    end
-
-    def find_td_import_jar
-      jar = find_files('td-import.jar', [jarfile_dest_path])
-      if jar.empty?
-        $stderr.puts "Cannot find td-import.jar in '#{jarfile_dest_path}'."
-        exit 10
+    else
+      puts 'Installed td-import.jar is already at the latest version.' unless hourly
+    end
+  end
+
+  def check_n_update_jar(hourly = false)
+    if hourly
+      if !ENV['TD_TOOLBELT_JAR_UPDATE'].nil?
+        # also validates the TD_TOOLBELT_JAR_UPDATE environment variable value
+        if ENV['TD_TOOLBELT_JAR_UPDATE'] == "0"
+          puts "Warning: Bulk Import JAR auto-update disabled by TD_TOOLBELT_JAR_UPDATE=0"
+          return
+        elsif ENV['TD_TOOLBELT_JAR_UPDATE'] != "1"
+          raise UpdateError,
+                "Invalid value for TD_TOOLBELT_JAR_UPDATE environment variable. Only 0 and 1 are allowed."
+        end
       end
-      jar.first
-    end
-
-    def find_logging_property
-      installed_path = File.join(File.expand_path('../..', File.dirname(__FILE__)), 'java')
 
-      config = find_files("logging.properties", [installed_path])
-      if config.empty?
-        puts "Cannot find 'logging.properties' file in '#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
-        []
-      else
-        config.first
+      if File.exists?(last_jar_autoupdate_timestamp) && \
+       (Time.now - File.mtime(last_jar_autoupdate_timestamp)).to_i < (60 * 60 * 1) # every hour
+        return
       end
     end
-
-  end # end of module Updater
-end # end of module TreasureData
+    jar_update(hourly)
+    FileUtils.touch last_jar_autoupdate_timestamp
+  end
+
+  private
+  def last_jar_autoupdate_timestamp
+    File.join(jarfile_dest_path, "td-import-java.version")
+  end
+
+  private
+  def existent_jar_updated_time
+    files = find_files("td-import-java.version", [jarfile_dest_path])
+    if files.empty?
+      return Time.at(0)
+    end
+    content = File.read(files.first)
+    index = content.index(' ')
+    time = nil
+    if index.nil?
+      time = Time.at(0).utc
+    else
+      time = Time.parse(content[index+1..-1].strip).utc
+    end
+    time
+  end
+
+  #
+  # Helpers
+  #
+  def find_files(glob, locations)
+    files = []
+    locations.each {|loc|
+      files = Dir.glob("#{loc}/#{glob}")
+      break unless files.empty?
+    }
+    files
+  end
+
+  def find_version_file
+    version = find_files('VERSION', [jarfile_dest_path])
+    if version.empty?
+      $stderr.puts "Cannot find VERSION file in '#{jarfile_dest_path}'."
+      exit 10
+    end
+    version.first
+  end
+
+  def find_td_import_jar
+    jar = find_files('td-import.jar', [jarfile_dest_path])
+    if jar.empty?
+      $stderr.puts "Cannot find td-import.jar in '#{jarfile_dest_path}'."
+      exit 10
+    end
+    jar.first
+  end
+
+  def find_logging_property
+    installed_path = File.join(File.expand_path('../..', File.dirname(__FILE__)), 'java')
+
+    config = find_files("logging.properties", [installed_path])
+    if config.empty?
+      puts "Cannot find 'logging.properties' file in '#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+      []
+    else
+      config.first
+    end
+  end
+
+end # module Updater
+end # module TreasureData