td 0.10.98 → 0.10.99

data/ChangeLog

@@ -1,8 +1,15 @@
+== 2014-02-26 version 0.10.99
+
+* job:show, query: limit the number of records ouputted when printing on stdout using the -l / --limit option
+* job:show, query: optionally output/store csv and tsv files with headers using the -c / --column-names option
+* job:show, query: support result output from Presto
+
 
 == 2014-02-21 version 0.10.98
 
 * import:unfreeze: fixed NoMethodError error
 * import:show uses more efficient REST API
+* Update td-client-ruby to 0.8.57
 
 
 == 2013-12-18 version 0.10.97

data/README.rdoc

@@ -37,6 +37,7 @@ Disable RVM or rbenv and use ruby.pkg's ruby (/usr/local/td/ruby/bin/ruby).
 And then run following commands:
 
     $ /usr/local/td/ruby/bin/gem install bundler rubyzip
+    $ /usr/local/td/ruby/bin/bundle install
     $ /usr/local/td/ruby/bin/rake pkg:build
 
 == Windows
@@ -59,5 +60,5 @@ Then run following commands on MinGW Shell:
 
 = Copyright
 
-Copyright:: Copyright (c) 2011 Treasure Data Inc.
+Copyright:: Copyright (c) 2014 Treasure Data Inc.
 License::   Apache License, Version 2.0

data/lib/td/command/job.rb

@@ -47,7 +47,7 @@ module Command
     op.on('-E', '--error', 'show only failed jobs', TrueClass) {|b|
       status = 'error'
     }
-    op.on('--slow [SECONDS]', 'show slow queries (default threshold: 3600 seconds)', Integer) { |i|
+    op.on('--slow [SECONDS]', 'show slow queries (default threshold: 3600 seconds)', Integer) {|i|
       slower_than = i || 3600
     }
     set_render_format_option(op)
@@ -85,7 +85,8 @@ module Command
     wait = false
     output = nil
     format = nil
-    render_opts = {}
+    render_opts = {:header => false}
+    limit = nil
     exclude = false
 
     op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
@@ -107,18 +108,40 @@ module Command
       end
       format = s
     }
+    op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s|
+      unless s.to_i > 0
+        raise "Invalid limit number. Must be a positive integer"
+      end
+      limit = s.to_i
+    }
+    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to tsv and csv formats)', TrueClass) {|b|
+      render_opts[:header] = b;
+    }
     op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
       exclude = b
     }
 
     job_id = op.cmd_parse
 
+    # parameter concurrency validation
+
     if output.nil? && format
       unless ['tsv', 'csv', 'json'].include?(format)
-        raise "Supported formats are only tsv, csv and json without --output option"
+        raise "Supported formats are only tsv, csv and json without -o / --output option"
       end
     end
 
+    if render_opts[:header]
+      unless ['tsv', 'csv'].include?(format)
+        raise "Option -c / --column-header is only supported with tsv and csv formats"
+      end
+    end
+
+    if !output.nil? && !limit.nil?
+      raise "Option -l / --limit is only valid when not outputting to file " +
+            "(no -o / --output option provided)"
+    end
+
     client = get_client
 
     job = client.job(job_id)
@@ -127,43 +150,51 @@ module Command
     #puts "URL         : #{job.url}"
     puts "Status      : #{job.status}"
     puts "Type        : #{job.type}"
-    puts "Priority    : #{job_priority_name_of(job.priority)}"
-    puts "Retry limit : #{job.retry_limit}"
-    puts "Result      : #{job.result_url}"
     puts "Database    : #{job.db_name}"
-    puts "Query       : #{job.query}"
+    # exclude some fields from bulk_import_perform type jobs
+    if [:hive, :pig, :impala, :presto].include?(job.type)
+      puts "Priority    : #{job_priority_name_of(job.priority)}"
+      puts "Retry limit : #{job.retry_limit}"
+      puts "Output      : #{job.result_url}"
+      puts "Query       : #{job.query}"
+    end
 
     if wait && !job.finished?
       wait_job(job)
-      if [:hive, :pig, :impala].include?(job.type) && !exclude
+      if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude
         puts "Result      :"
         begin
-          show_result(job, output, format, render_opts)
+          show_result(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e
           # Got 404 because result not found.
         end
       end
 
     else
-      if [:hive, :pig, :impala].include?(job.type) && !exclude
+      if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude
         puts "Result      :"
         begin
-          show_result(job, output, format, render_opts)
+          show_result(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e
+          # Got 404 because result not found.
         end
       end
 
       if verbose
-        puts ""
-        puts "cmdout:"
-        job.debug['cmdout'].to_s.split("\n").each {|line|
-          puts "  "+line
-        }
-        puts ""
-        puts "stderr:"
-        job.debug['stderr'].to_s.split("\n").each {|line|
-          puts "  "+line
-        }
+        if !job.debug['cmdout'].nil?
+          puts ""
+          puts "cmdout:"
+          job.debug['cmdout'].to_s.split("\n").each {|line|
+            puts "  " + line
+          }
+        end
+        if !job.debug['stderr'].nil?
+          puts ""
+          puts "stderr:"
+          job.debug['stderr'].to_s.split("\n").each {|line|
+            puts "  " + line
+          }
+        end
       end
     end
 
@@ -226,72 +257,97 @@ module Command
     end
   end
 
-  def show_result(job, output, format, render_opts={})
+  def show_result(job, output, limit, format, render_opts={})
     if output
-      write_result(job, output, format)
+      write_result(job, output, limit, format, render_opts)
       puts "written to #{output} in #{format} format"
     else
-      render_result(job, render_opts, format)
+      render_result(job, limit, format, render_opts)
     end
   end
 
-  def write_result(job, output, format)
+  def write_result(job, output, limit, format, render_opts={})
+
+    # the next 3 formats allow writing to both a file and stdout
+
     case format
     when 'json'
       require 'yajl'
-      first = true
-      open_file(output, "w") { |f|
+      open_file(output, "w") {|f|
         f.write "["
+        n_rows = 0
         job.result_each {|row|
-          if first
-            first = false
-          else
-            f.write ","
-          end
+          f.write ",\n" if n_rows > 0
           f.write Yajl.dump(row)
+          n_rows += 1
+          break if output.nil? and !limit.nil? and n_rows == limit
         }
         f.write "]"
       }
       puts if output.nil?
 
-    when 'msgpack'
-      open_file(output, "wb") { |f|
-        job.result_format('msgpack', f)
-      }
-
-    when 'msgpack.gz'
-      open_file(output, "wb") { |f|
-        job.result_format('msgpack.gz', f)
-      }
-
     when 'csv'
       require 'yajl'
       require 'csv'
 
-      open_file(output, "w") { |f|
+      open_file(output, "w") {|f|
         writer = CSV.new(f)
+        n_rows = 0
+        # output headers
+        if render_opts[:header] && job.hive_result_schema
+          writer << job.hive_result_schema.map {|name,type|
+            name
+          }
+        end
+        # output data
         job.result_each {|row|
-          writer << row.map {|col| dump_column(col) }
+          # TODO limit the # of columns
+          writer << row.map {|col|
+            dump_column(col)
+          }
+          n_rows += 1
+          break if output.nil? and !limit.nil? and n_rows == limit
         }
       }
 
     when 'tsv'
       require 'yajl'
-      open_file(output, "w") { |f|
+      open_file(output, "w") {|f|
+        # output headers
+        if render_opts[:header] && job.hive_result_schema
+          job.hive_result_schema.each {|name,type|
+            f.write name + "\t"
+          }
+          f.write "\n"
+        end
+        # output data
+        n_rows = 0
         job.result_each {|row|
-          first = true
+          n_cols = 0
           row.each {|col|
-            if first
-              first = false
-            else
-              f.write "\t"
-            end
+            f.write "\t" if n_cols > 0
+            # TODO limit the # of columns
             f.write dump_column(col)
+            n_cols += 1
           }
           f.write "\n"
+          n_rows += 1
+          break if output.nil? and !limit.nil? and n_rows == limit
         }
       }
 
+    # these last 2 formats are only valid if writing the result to file through the -o/--output option.
+
+    when 'msgpack'
+      open_file(output, "wb") {|f|
+        job.result_format('msgpack', f)
+      }
+
+    when 'msgpack.gz'
+      open_file(output, "wb") {|f|
+        job.result_format('msgpack.gz', f)
+      }
+
     else
       raise "Unknown format #{format.inspect}"
     end
@@ -311,26 +367,32 @@ module Command
     end
   end
 
-  def render_result(job, opts, format = nil)
+  def render_result(job, limit, format=nil, render_opts={})
     require 'yajl'
 
     if format.nil?
+      # display result in tabular format
       rows = []
+      n_rows = 0
       job.result_each {|row|
         # TODO limit number of rows to show
         rows << row.map {|v|
           dump_column(v)
         }
+        n_rows += 1
+        break if !limit.nil? and n_rows == limit
       }
 
-      opts[:max_width] = 10000
+      render_opts[:max_width] = 10000
       if job.hive_result_schema
-        opts[:change_fields] = job.hive_result_schema.map {|name,type| name }
+        render_opts[:change_fields] = job.hive_result_schema.map { |name,type| name }
       end
 
-      puts cmd_render_table(rows, opts)
+      puts cmd_render_table(rows, render_opts)
     else
-      write_result(job, nil, format)
+      # display result in any of: json, csv, tsv.
+      # msgpack and mspgpack.gz are not supported for stdout output
+      write_result(job, nil, limit, format, render_opts)
     end
   end
 

data/lib/td/command/query.rb

@@ -7,7 +7,7 @@ module Command
     wait = false
     output = nil
     format = nil
-    render_opts = {}
+    render_opts = {:header => false}
     result_url = nil
     result_user = nil
     result_ask_password = false
@@ -16,6 +16,7 @@ module Command
     query = nil
     sampling_all = nil
     type = nil
+    limit = nil
     exclude = false
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
@@ -58,24 +59,41 @@ module Command
     op.on('-q', '--query PATH', 'use file instead of inline query') {|s|
       query = File.open(s) { |f| f.read.strip }
     }
-    op.on('-T', '--type TYPE', 'set query type (hive or pig)') {|s|
+    op.on('-T', '--type TYPE', 'set query type (hive, pig, impala, presto)') {|s|
       type = s.to_sym
     }
     op.on('--sampling DENOMINATOR', 'enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|
       sampling_all = i
     }
+    op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s|
+      unless s.to_i > 0
+        raise "Invalid limit number. Must be a positive integer"
+      end
+      limit = s.to_i
+    }
+    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to tsv and csv formats)', TrueClass) {|b|
+      render_opts[:header] = b;
+    }
     op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
       exclude = b
     }
 
     sql = op.cmd_parse
 
+    # parameter concurrency validation
+
     if output.nil? && format
       unless ['tsv', 'csv', 'json'].include?(format)
         raise "Supported formats are only tsv, csv and json without --output option"
       end
     end
 
+    if render_opts[:header]
+      unless ['tsv', 'csv'].include?(format)
+        raise "Option -c / --column-header is only supported with tsv and csv formats"
+      end
+    end
+
     unless db_name
       $stderr.puts "-d, --database DB_NAME option is required."
       exit 1
@@ -117,7 +135,7 @@ module Command
       if job.success? && !exclude
         puts "Result     :"
         begin
-          show_result(job, output, format, render_opts)
+          show_result(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e
         end
       end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.98'
+VERSION = '0.10.99'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.98
+  version: 0.10.99
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-22 00:00:00.000000000 Z
+date: 2014-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -288,12 +288,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 687370063067596647
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 687370063067596647
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -310,4 +316,3 @@ test_files:
 - spec/spec_helper.rb
 - spec/td/helpers_spec.rb
 - spec/td/version_spec.rb
-has_rdoc: false