td 0.11.1 → 0.11.2

data/lib/td/command/db.rb

@@ -41,7 +41,7 @@ module Command
 
     if dbs.empty?
       $stderr.puts "There are no databases."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db_name>' to create a database."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create <db_name>' to create a database."
     end
   end
 
@@ -61,7 +61,7 @@ module Command
     end
 
     $stderr.puts "Database '#{db_name}' is created."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create #{db_name} <table_name>' to create a table."
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create #{db_name} <table_name>' to create a table."
   end
 
   def db_delete(op)

data/lib/td/command/export.rb

@@ -65,7 +65,7 @@ module Command
     job = client.export(db_name, table_name, "s3", s3_opts)
 
     $stderr.puts "Export job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
   end
 
   private

data/lib/td/command/import.rb

@@ -150,7 +150,16 @@ module Command
 
   private
   def set_sysprops_endpoint(sysprops)
-    endpoint = ENV['TD_API_SERVER']
+    # optional, if not provided a default is used from the ruby client library
+    begin
+      endpoint = Config.endpoint
+    rescue ConfigNotFoundError => e
+      # rescue the ConfigNotFoundError exception which originates when
+      #   the config file is not found or an endpoint is not provided
+      #   either through -e / --endpoint or TD_API_SERVER environment
+      #   variable.
+    end
+
     if endpoint
       require 'uri'
 
@@ -160,23 +169,31 @@ module Command
       when 'http', 'https'
         host = uri.host
         port = uri.port
-        ssl = uri.scheme == 'https'
-
-        port = 80 if port == 443 and ssl
+        # NOTE: Config.secure option is ignored in favor
+        # of defining ssl based on the URL scheme
+        ssl = (uri.scheme == 'https')
       else
-        if uri.port
-          # invalid URI
+        # uri scheme is not set if the endpoint is
+        #   like 'api.treasuredata.com:80'. In that case the
+        #   uri object does not contain any useful info.
+        if uri.port # invalid URI
           raise "Invalid endpoint: #{endpoint}"
         end
 
         # generic URI
         host, port = endpoint.split(':', 2)
         port = port.to_i
-        # TODO support ssl
-        port = 80 if port == 0
-        ssl = false
-      end
+        # Config.secure = false is the --insecure option was used
+        if Config.secure
+          port = 443 if port == 0
+          ssl = true
+        else
+          port = 80 if port == 0
+          ssl = false
+        end
+        end
 
+      sysprops << "-Dtd.api.server.scheme=#{ssl ? 'https' : 'http'}://"
       sysprops << "-Dtd.api.server.host=#{host}"
       sysprops << "-Dtd.api.server.port=#{port}"
     end
@@ -184,17 +201,30 @@ module Command
 
   private
   def set_sysprops_http_proxy(sysprops)
-    http_proxy = ENV['HTTP_PROXY']
-    if http_proxy
-      if http_proxy =~ /\Ahttp:\/\/(.*)\z/
-        http_proxy = $~[1]
+    uri_string = ENV['HTTP_PROXY']
+    return unless uri_string
+
+    require 'uri'
+    uri = URI.parse(uri_string)
+    proxy_host = nil
+    proxy_port = nil
+
+    case uri.scheme
+    when 'http'
+      if uri.host
+        proxy_host = uri.host             # host is required
+        proxy_port = uri.port if uri.port # default value of uri.port is 80.
       end
-      proxy_host, proxy_port = http_proxy.split(':', 2)
+    when 'https'
+      raise ParameterConfigurationError,
+            "HTTP proxy URL must use 'http' protocol. Example format: 'http://localhost:3128'."
+    else
+      proxy_host, proxy_port = uri_string.split(':', 2)
       proxy_port = (proxy_port ? proxy_port.to_i : 80)
-
-      sysprops << "-Dhttp.proxyHost=#{proxy_host}"
-      sysprops << "-Dhttp.proxyPort=#{proxy_port}"
     end
+
+    sysprops << "-Dhttp.proxyHost=#{proxy_host}" if proxy_host
+    sysprops << "-Dhttp.proxyPort=#{proxy_port}" if proxy_port
   end
 
 end

data/lib/td/command/job.rb

@@ -72,8 +72,8 @@ module Command
     rows = []
     jobs.each {|job|
       start = job.start_at
-      elapsed = humanize_elapsed_time(start, job.end_at)
-      cpu_time = humanize_time(job.cpu_time, true)
+      elapsed = Command.humanize_elapsed_time(start, job.end_at)
+      cpu_time = Command.humanize_time(job.cpu_time, true)
       priority = job_priority_name_of(job.priority)
       rows << {
         :JobID => job.job_id,
@@ -181,7 +181,7 @@ module Command
     end
     # if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
     if job.finished? && [:hive].include?(job.type)
-      puts "CPU time    : #{humanize_time(job.cpu_time, true)}"
+      puts "CPU time    : #{Command.humanize_time(job.cpu_time, true)}"
     end
 
     if wait && !job.finished?
@@ -196,7 +196,7 @@ module Command
       end
 
     else
-      if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude
+      if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude && job.finished?
         puts "Result      :"
         begin
           show_result(job, output, limit, format, render_opts)
@@ -294,26 +294,6 @@ module Command
 
   def write_result(job, output, limit, format, render_opts={})
 
-    # start progress indicator
-    line_len = 0
-    start_time = last_time = Time.now.to_i
-    base_msg = "WARNING: the query result is being written"
-    if !output.nil?
-      msg = base_msg + "\r"
-      line_len += msg.length
-      print msg
-    end
-
-    # this lambda is passed on as a block when outputing the results to file
-    progress = lambda {
-      if (time = Time.now.to_i) - last_time > 5
-        msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
-        line_len = msg.length
-        print msg
-        last_time = time
-      end
-    }
-
     # the next 3 formats allow writing to both a file and stdout
 
     case format
@@ -327,9 +307,6 @@ module Command
           f.write Yajl.dump(row)
           n_rows += 1
           break if output.nil? and !limit.nil? and n_rows == limit
-
-          # update progress indicator
-          progress.call if !output.nil?
         }
         f.write "]"
       }
@@ -357,9 +334,6 @@ module Command
           n_rows += 1
           writer.flush if n_rows % 100 == 0 # flush every 100 recods
           break if output.nil? and !limit.nil? and n_rows == limit
-
-          # update progress indicator
-          progress.call if !output.nil?
         }
       }
 
@@ -388,9 +362,6 @@ module Command
           n_rows += 1
           f.flush if n_rows % 100 == 0 # flush every 100 recods
           break if output.nil? and !limit.nil? and n_rows == limit
-
-          # update progress indicator
-          progress.call if !output.nil?
         }
       }
 
@@ -402,7 +373,7 @@ module Command
               "Format 'msgpack' does not support writing to stdout"
       end
       open_file(output, "wb") {|f|
-        job.result_format('msgpack', f, &progress)
+        job.result_format('msgpack', f)
       }
 
     when 'msgpack.gz'
@@ -411,18 +382,12 @@ module Command
               "Format 'msgpack' does not support writing to stdout"
       end
       open_file(output, "wb") {|f|
-        job.result_format('msgpack.gz', f, &progress)
+        job.result_format('msgpack.gz', f)
       }
 
     else
       raise "Unknown format #{format.inspect}"
     end
-
-    # clear the progress indicator
-    if !output.nil?
-      print "\r"
-      print " " * (line_len + 5) + "\r"
-    end
   end
 
   def open_file(output, mode)
@@ -443,17 +408,10 @@ module Command
     require 'yajl'
 
     if format.nil?
-
-      # start progress indicator
-      start_time = last_time = Time.now.to_i
-      base_msg = "WARNING: the query result is being downloaded"
-      msg = base_msg + "\r"
-      line_len = msg.length
-      print msg
-
       # display result in tabular format
       rows = []
       n_rows = 0
+      print "WARNING: the query result is being downloaded...\r"
       job.result_each {|row|
         # TODO limit number of rows to show
         rows << row.map {|v|
@@ -461,20 +419,8 @@ module Command
         }
         n_rows += 1
         break if !limit.nil? and n_rows == limit
-
-        # progress indication
-        time = Time.now.to_i
-        if time - last_time > 5
-          msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
-          line_len = msg.length
-          print msg
-          last_time = time
-        end
       }
-
-      # clear the progress indicator
-      print "\r"
-      print " " * (line_len + 5) + "\r"
+      print " " * 100, "\r" # make sure the previous WARNING is cleared over
 
       render_opts[:max_width] = 10000
       if job.hive_result_schema
@@ -512,6 +458,7 @@ module Command
     }
     return nil
   end
-end
-end
+
+end # module Command
+end # module TrasureData
 

data/lib/td/command/list.rb

@@ -1,4 +1,3 @@
-
 module TreasureData
 module Command
 module List
@@ -46,7 +45,7 @@ module List
 
     def cmd_usage(msg=nil)
       puts self.to_s
-      puts "error: #{msg}" if msg
+      puts "Error: #{msg}" if msg
       exit 1
     end
   end
@@ -309,6 +308,7 @@ module List
   # TODO acl:test
 
   add_list 'server:status', %w[], 'Show status of the Treasure Data server'
+  add_list 'server:endpoint', %w[api_endpoint], "Set the Treasure Data API server's endpoint (must be a valid URI)", "td server:endpoint 'https://api.treasuredata.com'"
 
   add_list 'sample:apache', %w[path.json], 'Create a sample log file'
 
@@ -396,7 +396,8 @@ module List
   add_guess 'server-status',    'server:status'
 
   finishup
-end
-end
-end
+
+end # module List
+end # module Command
+end # module TreasureData
 

data/lib/td/command/query.rb

@@ -119,9 +119,6 @@ module Command
     if result_url
       require 'td/command/result'
       result_url = build_result_url(result_url, result_user, result_ask_password)
-      if result_url =~ /^td:/
-        validate_td_result_url(result_url)
-      end
     end
 
     client = get_client
@@ -134,7 +131,7 @@ module Command
     job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
 
     puts "Job #{job.job_id} is queued."
-    puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
+    puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
     #puts "See #{job.url} to see the progress."
 
     if wait

data/lib/td/command/result.rb

@@ -11,7 +11,7 @@ module Command
 
     unless r
       $stderr.puts "Result URL '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "result:create #{name} <URL>' to create the URL."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create #{name} <URL>' to create the URL."
       exit 1
     end
 
@@ -40,7 +40,7 @@ module Command
 
     if rs.empty?
       $stderr.puts "There are no result URLs."
-      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "result:create <name> <url>' to create a result URL."
+      $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create <name> <url>' to create a result URL."
     end
   end
 
@@ -123,9 +123,11 @@ module Command
     url
   end
 
+  # DEPRECATED: relying on API server side validation which will return
+  #             immediately after query submission with error code 422.
   private
   def validate_td_result_url(url)
-    re = /td:\/\/[^@]*@\/(.*)\/(.*)?/
+    re = /td:\/\/[^@]*@\/(.*)\/([^?]+)/
     match = re.match(url)
     if match.nil?
       raise ParameterConfigurationError, "Treasure Data result output invalid URL format"

data/lib/td/command/runner.rb

@@ -1,4 +1,3 @@
-
 module TreasureData
 module Command
 
@@ -6,11 +5,12 @@ class Runner
   def initialize
     @config_path = nil
     @apikey = nil
+    @endpoint = nil
     @prog_name = nil
-    @secure = true
+    @insecure = false
   end
 
-  attr_accessor :apikey, :config_path, :prog_name, :secure
+  attr_accessor :apikey, :endpoint, :config_path, :prog_name, :insecure
 
   def run(argv=ARGV)
     require 'td/version'
@@ -58,7 +58,7 @@ Additional commands:
 Type 'td help COMMAND' for more information on a specific command.
 EOF
         if errmsg
-          puts "error: #{errmsg}"
+          puts "Error: #{errmsg}"
           exit 1
         else
           exit 0
@@ -66,13 +66,16 @@ EOF
       end
     end
 
+    # there local vars are loaded with the values of the options below
+    # here they are preloaded with the defaults
     config_path = @config_path
     apikey = @apikey
+    endpoint = @endpoint
     insecure = nil
     $verbose = false
     #$debug = false
 
-    op.on('-c', '--config PATH', "path to config file (~/.td/td.conf)") {|s|
+    op.on('-c', '--config PATH', "path to the configuration file (default: ~/.td/td.conf)") {|s|
       config_path = s
     }
 
@@ -80,7 +83,15 @@ EOF
       apikey = s
     }
 
-    op.on('--insecure', "Insecure access: disable SSL") { |b|
+    op.on('-e', '--endpoint API_SERVER', "specify the URL for API server to use (default: https://api.treasuredata.com)." ,
+                                         "  The URL must contain a scheme (http:// or https:// prefix) to be valid.",
+                                         "  Valid IPv4 addresses are accepted as well in place of the host name.") {|e|
+      require 'td/command/common'
+      Command.validate_api_endpoint(e)
+      endpoint = e
+    }
+
+    op.on('--insecure', "Insecure access: disable SSL (enabled by default)") {|b|
       insecure = true
     }
 
@@ -106,6 +117,8 @@ EOF
       usage nil if argv.empty?
       cmd = argv.shift
 
+      # NOTE: these information are loaded from by each command through
+      #       'TreasureData::Command::get_client' from 'lib/td/command/common.rb'
       require 'td/config'
       if config_path
         Config.path = config_path
@@ -114,6 +127,10 @@ EOF
         Config.apikey = apikey
         Config.cl_apikey = true
       end
+      if endpoint
+        Config.endpoint = endpoint
+        Config.cl_endpoint = true
+      end
       if insecure
         Config.secure = false
       end
@@ -145,8 +162,13 @@ EOF
       # List of new exceptions:
       # => ParameterConfigurationError
       # => BulkImportExecutionError
-      unless [ParameterConfigurationError, BulkImportExecutionError].include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
-        $stderr.puts "error #{$!.class}: backtrace:"
+      # => UpUpdateError
+      require 'td/client/api'
+      # => APIError
+      unless [ParameterConfigurationError, BulkImportExecutionError, UpdateError,
+              APIError]
+              .include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
+        $stderr.puts "Error #{$!.class}: backtrace:"
         $!.backtrace.each {|b|
           $stderr.puts "  #{b}"
         }
@@ -167,9 +189,8 @@ EOS
     end
     return 0
   end
-end
-
+end # class Runner
 
-end
-end
+end # module Command
+end # module TreasureData