td 0.11.3 → 0.11.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MTZmYjMxNzU1OTk3MzQ0MzcyMGU5NzRhNzdkZjZhM2Q4ZTFlNDEyMw==
+    NTM4MDYwMmE3YThiODA3Njc0ZmVhMDA4ODY5Yjk5NmZiOWNiZDFhNQ==
   data.tar.gz: !binary |-
-    NzFhNTY5YjIyNmRlYTUzNzk1YWIxY2Q1MzMxMzc1YjFhODkxZmY4ZA==
+    MzM2MDI0MjAxZjc0Mjk2M2QwZDEyYTM1Y2NmNzhlYzkyY2YyNDBmYQ==
 SHA512:
   metadata.gz: !binary |-
-    MTZkZGQyMWFkYjcxMDE2NWI2ZWEwZTVjNDQ5ODQ4YzYxMjMwOTJmZTI1Njc4
-    MGI4YWU1NzgzOTRmNjQ3ZDE5YjUxMTY2ZjhlMDM5Y2MxMmFjMWE4ZTEwNWM3
-    ZmZlYzM5ODRmYTQ0MTE0ZmRkMmFmNTNmNGYyMDU5MDhjNzRhOGE=
+    MGQ3YWY3NTRhMDJmNzU4OWMwY2M2YmY0ODliZTAwM2E3MTIwZGVkYjUxNWVk
+    Y2ZiYjhiNDQxOGQ2YjExZTg0ZDU2NjRiZTQ4YTZmY2E1YTc2YWQzZDI0MGU3
+    ZWM1OGZhZGU1NzQ2NTgzNzYyOTQwZWFjNzdiYmE0M2E5ZGJlZDg=
   data.tar.gz: !binary |-
-    OWI1OGZjYTE4ODBjZDc4YWFiMDJhZWNmYmVlNjg3OTdkODA4Y2FlMjkyMDY5
-    OGY3M2M1N2QzYTMwMGFiZDhhM2RhZTQ3ODVkMjgxNWE0MGYzZjFhM2Y3Y2Ux
-    OGZiYTU0ZTFiOWEyZGYzZTgyMmRmMTFhOTY0Mzc2ZDM4Y2NlN2Q=
+    MzRlMTE0NDdiNjE5YmMzNGEyN2MyZjRmNDgyNjIxYjU1MjllNTc2YjA5YmJh
+    OTQ4OTY2M2QxNjEzNDQ3NThjMTE2NzdiZjFhODU5NGJjY2U1YzNjOGRjYmY2
+    NDU1NDZlMzNkMTgwNDlhZDRhOWRhOGQ1Y2Y1OGE2NTliZTc5ZmI=

data/ChangeLog

@@ -1,3 +1,13 @@
+== 2014-09-25 version 0.11.4
+
+* Expand file path containing wildcards and not escaped by the underlying shell
+* Add a global '-r / --retry-post-requests' option to enable the td-client-ruby
+  retrying mechanism for post request. 
+  Note: -r option must be right after 'td' before subcommands like: td -r query
+  WARNING: enabling this option can lead to unexpected results because the
+  retrying mechanism can caused the creation of duplicated resources since the
+  POST methods are not idempotent.
+
 == 2014-07-31 version 0.11.3
 
 * Support name updates for schedules using sched:update command

data/lib/td/command/common.rb

@@ -32,6 +32,9 @@ module Command
     unless opts.has_key?(:ssl)
       opts[:ssl] = Config.secure
     end
+    unless opts.has_key?(:retry_post_requests)
+      opts[:retry_post_requests] = Config.retry_post_requests
+    end
 
     # apikey is mandatory
     apikey = Config.apikey
@@ -373,17 +376,23 @@ EOS
     end
 
     def update(curr_size)
-      ratio = (curr_size.to_f * 100 / @size).round(1)
-      if ratio >= (@last_perc_step + @perc_step) &&
-         (!@min_periodicity || (time = Time.now.to_i) - @last_time >= @min_periodicity)
-        msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)} / #{ratio}%"
-        print "\r" + " " * (msg.length + 10)
+      if @size.nil? || @size == 0
+        msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)}"
         print msg
-        @last_perc_step = ratio
-        @last_time = time
         true
       else
-        false
+        ratio = (curr_size.to_f * 100 / @size).round(1)
+        if ratio >= (@last_perc_step + @perc_step) &&
+           (!@min_periodicity || (time = Time.now.to_i) - @last_time >= @min_periodicity)
+          msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)} / #{ratio}%"
+          print "\r" + " " * (msg.length + 10)
+          print msg
+          @last_perc_step = ratio
+          @last_time = time
+          true
+        else
+          false
+        end
       end
     end
 

data/lib/td/command/job.rb

@@ -116,6 +116,9 @@ module Command
       render_opts[:vertical] = b
     }
     op.on('-o', '--output PATH', 'write result to the file') {|s|
+      unless Dir.exist?(File.dirname(s))
+        s = File.expand_path(s)
+      end
       output = s
       format = 'tsv' if format.nil?
     }
@@ -190,14 +193,34 @@ module Command
       end
     end
 
+    # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
+    retry_delay = 5
+    max_cumul_retry_delay = 200
+    cumul_retry_delay = 0
+
     if wait && !job.finished?
       wait_job(job)
       if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude
         puts "Result      :"
+
         begin
           show_result(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e
           # Got 404 because result not found.
+        rescue TreasureData::APIError, # HTTP status code 500 or more
+               Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError => e
+          # don't retry on 300 and 400 errors
+          if e.class == TreasureData::APIError && e.message !~ /^5\d\d:\s+/
+            raise e
+          end
+          if cumul_retry_delay > max_cumul_retry_delay
+            raise e
+          end
+          $stderr.puts "Error #{e.class}: #{e.message}. Retrying after #{retry_delay} seconds..."
+          sleep retry_delay
+          cumul_retry_delay += retry_delay
+          retry_delay *= 2
+          retry
         end
       end
 
@@ -208,6 +231,20 @@ module Command
           show_result(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e
           # Got 404 because result not found.
+        rescue TreasureData::APIError,
+               Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError => e
+          # don't retry on 300 and 400 errors
+          if e.class == TreasureData::APIError && e.message !~ /^5\d\d:\s+/
+            raise e
+          end
+          if cumul_retry_delay > max_cumul_retry_delay
+            raise e
+          end
+          $stderr.puts "Error #{e.class}: #{e.message}. Retrying after #{retry_delay} seconds..."
+          sleep retry_delay
+          cumul_retry_delay += retry_delay
+          retry_delay *= 2
+          retry
         end
       end
 
@@ -229,7 +266,7 @@ module Command
       end
     end
 
-    puts "Use '-v' option to show detailed messages." unless verbose
+    puts "\rUse '-v' option to show detailed messages." + " " * 20 unless verbose
   end
 
   def job_status(op)

data/lib/td/command/list.rb

@@ -303,7 +303,7 @@ module List
   add_list 'user:apikey:list', %w[name], 'Show API keys of an user'
   add_list 'user:apikey:add', %w[name], 'Add an API key to an user'
   add_list 'user:apikey:remove', %w[name apikey], 'Remove an API key from an user'
-  add_list 'user:password:change', %w[name], 'Change password'
+  add_list 'user:password:change', %w[name], 'Change password for any user (must be an Administrator)'
 
   add_list 'acl:list', %w[], 'Show list of access controls'
   add_list 'acl:grant', %w[subject action scope], 'Grant an access control'

data/lib/td/command/query.rb

@@ -1,4 +1,3 @@
-
 module TreasureData
 module Command
 
@@ -28,6 +27,9 @@ module Command
       render_opts[:vertical] = b
     }
     op.on('-o', '--output PATH', 'write result to the file') {|s|
+      unless Dir.exist?(File.dirname(s))
+        s = File.expand_path(s)
+      end
       output = s
       format = 'tsv' if format.nil?
     }

data/lib/td/command/runner.rb

@@ -74,6 +74,7 @@ EOF
     insecure = nil
     $verbose = false
     #$debug = false
+    retry_post_requests = false
 
     op.on('-c', '--config PATH', "path to the configuration file (default: ~/.td/td.conf)") {|s|
       config_path = s
@@ -107,6 +108,12 @@ EOF
       return usage nil
     }
 
+    op.on('-r', '--retry-post-requests', "retry on failed post requests.",
+                                         "Warning: can cause resource duplication, such as duplicated job submissions.",
+                                         TrueClass) {|b|
+      retry_post_requests = b
+    }
+
     op.on('--version', "show version") {
       puts op.version
       return 0
@@ -134,6 +141,9 @@ EOF
       if insecure
         Config.secure = false
       end
+      if retry_post_requests
+        Config.retry_post_requests = true
+      end
     rescue
       return usage $!.to_s
     end

data/lib/td/command/table.rb

@@ -349,7 +349,9 @@ module Command
     to = nil
     wait = false
 
-    op.on('-t', '--to TIME', 'end time of logs to delete in Unix time multiple of 3600 (1 hour)') {|s|
+    op.on('-t', '--to TIME', 'end time of logs to delete in Unix time multiple of 3600 (1 hour)',
+                             '  or Ruby time string format (e.g. \'2014-07-01 14:00:00 JST\') where',
+                             '  the minutes and seconds are required to be 0.') {|s|
       if s.to_i.to_s == s
         # UNIX time
         to = s.to_i
@@ -358,7 +360,9 @@ module Command
         to = Time.parse(s).to_i
       end
     }
-    op.on('-f', '--from TIME', 'start time of logs to delete in Unix time multiple of 3600 (1 hour)') {|s|
+    op.on('-f', '--from TIME', 'start time of logs to delete in Unix time multiple of 3600 (1 hour)',
+                               '  or Ruby time string format (e.g. \'2014-07-01 13:00:00 JST\') where',
+                               '  the minutes and seconds are required to be 0.') {|s|
       if s.to_i.to_s == s
         from = s.to_i
       else
@@ -383,7 +387,8 @@ module Command
     end
 
     if from % 3600 != 0 || to % 3600 != 0
-      $stderr.puts "Time for the -f / --from and -t / --to options must be a multiple of 3600 (1 hour)"
+      $stderr.puts "Time for the -f / --from and -t / --to options must either be a multiple of 3600 (1 hour)\n" +
+                   "  or be expressed in Ruby time string format where the minutes and seconds are 0"
       exit 1
     end
 

data/lib/td/config.rb

@@ -22,6 +22,7 @@ class Config
   @@endpoint = nil if @@endpoint == ""
   @@cl_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line
   @@secure = true
+  @@retry_post_requests = false
 
   def initialize
     @path = nil
@@ -117,6 +118,15 @@ class Config
   end
 
 
+  def self.retry_post_requests
+    @@retry_post_requests
+  end
+
+  def self.retry_post_requests=(retry_post_requests)
+    @@retry_post_requests = retry_post_requests
+  end
+
+
   def self.apikey
     @@apikey || Config.read['account.apikey']
   end

data/lib/td/updater.rb

@@ -11,7 +11,7 @@ module Updater
 
   def self.raise_error(message)
     # TODO: Replace better Exception class
-    raise RuntimeError.new(message)
+    raise Command::UpdateError, message
   end
 
   # copied from TreasureData::Helpers to avoid load issue.
@@ -132,8 +132,7 @@ module Updater
     when Net::HTTPSuccess then response.body
     when Net::HTTPRedirection then fetch(response['Location'])
     else
-      raise Command::UpdateError,
-            "An error occurred when fetching from '#{url}'."
+      raise_error "An error occurred when fetching from '#{url}'."
       response.error!
     end
   end
@@ -290,9 +289,9 @@ module Updater
         end
         return stream_fetch(response['Location'], binfile, &progress)
       else
-        raise Command::UpdateError,
-              "An error occurred when fetching from '#{uri}' " +
-              "(#{response.class.to_s}: #{response.message})."
+        raise_error
+          "An error occurred when fetching from '#{uri}' " +
+          "(#{response.class.to_s}: #{response.message})."
         return false
       end
     }

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.11.3'
+  TOOLBELT_VERSION = '0.11.4'
 end

data/td.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.63"
+  gem.add_dependency "td-client", "~> 0.8.64"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 0.11.4
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -102,14 +102,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.63
+        version: 0.8.64
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.63
+        version: 0.8.64
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement