td 0.10.99 → 0.11.1

data/lib/td/command/list.rb

@@ -248,8 +248,8 @@ module List
   add_list 'import:list', %w[], 'List bulk import sessions', 'import:list'
   add_list 'import:show', %w[name], 'Show list of uploaded parts', 'import:show'
   add_list 'import:create', %w[name db table], 'Create a new bulk import session to the the table', 'import:create logs_201201 example_db event_logs'
-  add_list 'import:java_version', %w[], 'Show version', 'import:java_version'
-  add_list 'import:jar_update', %w[], 'Update import jar', 'import:jar_update'
+  add_list 'import:jar_version', %w[], 'Show import jar version', 'import:jar_version'
+  add_list 'import:jar_update', %w[], 'Update import jar to the latest version', 'import:jar_update'
   add_list 'import:prepare', %w[files_], 'Convert files into part file format', 'import:prepare logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
   add_list 'import:upload', %w[name files_], 'Upload or re-upload files into a bulk import session', 'import:upload parts/* --parallel 4'
   add_list 'import:auto', %w[name files_], 'Upload files and automatically perform and commit the data', 'import:auto parts/* --parallel 4'
@@ -261,9 +261,9 @@ module List
   add_list 'import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'import:unfreeze logs_201201'
 
   add_list 'result:list', %w[], 'Show list of result URLs', 'result:list', 'results'
-  add_list 'result:show', %w[name], 'Describe information of a result URL', 'result mydb'
-  add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create mydb mysql://my-server/mydb'
-  add_list 'result:delete', %w[name], 'Delete a result URL', 'result:delete mydb'
+  add_list 'result:show', %w[name], 'Describe information of a result URL', 'result name'
+  add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create name mysql://my-server/mydb'
+  add_list 'result:delete', %w[name], 'Delete a result URL', 'result:delete name'
 
   add_list 'status', %w[], 'Show schedules, jobs, tables and results', 'status', 's'
 
@@ -346,8 +346,10 @@ module List
   add_alias 'scheds', 'sched:list'
   add_alias 'schedules', 'sched:list'
 
-  add_alias 'import',           'import:show'
-  add_alias 'imports',          'import:list'
+  add_alias 'import',              'import:show'
+  add_alias 'imports',             'import:list'
+  add_alias 'import:java_version', 'import:jar_version'
+
 
   add_alias 'bulk_import', 'bulk_import:show'
   add_alias 'bulk_imports', 'bulk_import:list'

data/lib/td/command/query.rb

@@ -14,7 +14,6 @@ module Command
     priority = nil
     retry_limit = nil
     query = nil
-    sampling_all = nil
     type = nil
     limit = nil
     exclude = false
@@ -38,7 +37,9 @@ module Command
       end
       format = s
     }
-    op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)') {|s|
+    op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)',
+                                       ' It is suggested for this option to be used with the -x / --exclude option to suppress printing',
+                                       ' of the query result to stdout or -o / --output to dump the query result into a file.') {|s|
       result_url = s
     }
     op.on('-u', '--user NAME', 'set user name for the result URL') {|s|
@@ -62,8 +63,10 @@ module Command
     op.on('-T', '--type TYPE', 'set query type (hive, pig, impala, presto)') {|s|
       type = s.to_sym
     }
-    op.on('--sampling DENOMINATOR', 'enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|
-      sampling_all = i
+    op.on('--sampling DENOMINATOR', 'OBSOLETE - enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|
+      puts "WARNING: the random sampling feature enabled through the '--sampling' option was removed and does no longer"
+      puts "         have any effect. It is left for backwards compatibility with older scripts using 'td'."
+      puts
     }
     op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s|
       unless s.to_i > 0
@@ -80,23 +83,11 @@ module Command
 
     sql = op.cmd_parse
 
-    # parameter concurrency validation
-
-    if output.nil? && format
-      unless ['tsv', 'csv', 'json'].include?(format)
-        raise "Supported formats are only tsv, csv and json without --output option"
-      end
-    end
-
-    if render_opts[:header]
-      unless ['tsv', 'csv'].include?(format)
-        raise "Option -c / --column-header is only supported with tsv and csv formats"
-      end
-    end
+    # required parameters
 
     unless db_name
-      $stderr.puts "-d, --database DB_NAME option is required."
-      exit 1
+      raise ParameterConfigurationError,
+            "-d / --database DB_NAME option is required."
     end
 
     if sql == '-'
@@ -104,30 +95,47 @@ module Command
     elsif sql.nil?
       sql = query
     end
-
     unless sql
-      $stderr.puts "<sql> argument or -q,--query PATH option is required."
-      exit 1
+      raise ParameterConfigurationError,
+            "<sql> argument or -q / --query PATH option is required."
+    end
+
+    # parameter concurrency validation
+
+    if output.nil? && format
+      unless ['tsv', 'csv', 'json'].include?(format)
+        raise ParameterConfigurationError,
+              "Supported formats are only tsv, csv and json without --output option"
+      end
+    end
+
+    if render_opts[:header]
+      unless ['tsv', 'csv'].include?(format)
+        raise ParameterConfigurationError,
+              "Option -c / --column-header is only supported with tsv and csv formats"
+      end
     end
 
     if result_url
       require 'td/command/result'
       result_url = build_result_url(result_url, result_user, result_ask_password)
+      if result_url =~ /^td:/
+        validate_td_result_url(result_url)
+      end
     end
 
     client = get_client
 
-    # local existance check
+    # local existence check
     get_database(client, db_name)
 
     opts = {}
-    opts['sampling_all'] = sampling_all if sampling_all
     opts['type'] = type if type
     job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
 
-    $stderr.puts "Job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
-    #$stderr.puts "See #{job.url} to see the progress."
+    puts "Job #{job.job_id} is queued."
+    puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
+    #puts "See #{job.url} to see the progress."
 
     if wait
       wait_job(job, true)

data/lib/td/command/result.rb

@@ -56,8 +56,7 @@ module Command
     }
 
     name, url = op.cmd_parse
-
-    API.validate_database_name(name)
+    API.validate_result_set_name(name)
 
     client = get_client
 
@@ -123,6 +122,23 @@ module Command
 
     url
   end
+
+  private
+  def validate_td_result_url(url)
+    re = /td:\/\/[^@]*@\/(.*)\/(.*)?/
+    match = re.match(url)
+    if match.nil?
+      raise ParameterConfigurationError, "Treasure Data result output invalid URL format"
+    end
+    dbs = match[1]
+    tbl = match[2]
+    begin
+      API.validate_name("Treasure Data result output destination database", 3, 256, dbs)
+      API.validate_name("Treasure Data result output destination table", 3, 256, tbl)
+    rescue ParameterValidationError => e
+      raise ParameterConfigurationError, e
+    end
+  end
 end
 end
 

data/lib/td/command/runner.rb

@@ -2,7 +2,6 @@
 module TreasureData
 module Command
 
-
 class Runner
   def initialize
     @config_path = nil
@@ -21,7 +20,7 @@ class Runner
     $prog = @prog_name || File.basename($0)
 
     op = OptionParser.new
-    op.version = VERSION
+    op.version = TOOLBELT_VERSION
     op.banner = <<EOF
 usage: #{$prog} [options] COMMAND [args]
 
@@ -45,11 +44,11 @@ Basic commands:
   import         # manage bulk import sessions (Java based fast processing)
   bulk_import    # manage bulk import sessions (Old Ruby-based implementation)
   result         # create/delete/list result URLs
+  sched          # create/delete/list schedules that run a query periodically
+  schema         # create/delete/modify schemas of tables
 
 Additional commands:
 
-  sched          # create/delete/list schedules that run a query periodically
-  schema         # create/delete/modify schemas of tables
   status         # show scheds, jobs, tables and results
   apikey         # show/set API key
   server         # show status of the Treasure Data server
@@ -141,12 +140,19 @@ EOF
       $stderr.puts "TreasureData account is not configured yet."
       $stderr.puts "Run '#{$prog} account' first."
     rescue => e
-      $stderr.puts "error #{$!.class}: backtrace:"
-      $!.backtrace.each {|b|
-        $stderr.puts "  #{b}"
-      }
-      puts ""
-      puts $!
+      # work in progress look ahead development: new exceptions are rendered as simple
+      # error messages unless the TD_TOOLBELT_DEBUG variable is not empty.
+      # List of new exceptions:
+      # => ParameterConfigurationError
+      # => BulkImportExecutionError
+      unless [ParameterConfigurationError, BulkImportExecutionError].include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
+        $stderr.puts "error #{$!.class}: backtrace:"
+        $!.backtrace.each {|b|
+          $stderr.puts "  #{b}"
+        }
+        puts ""
+      end
+      puts "Error: " + $!.to_s
 
       require 'socket'
       if e.is_a?(::SocketError)
@@ -157,7 +163,9 @@ If you want to use td command through a proxy,
 please set HTTP_PROXY environment variable (e.g. export HTTP_PROXY="host:port")
 EOS
       end
+      return 1
     end
+    return 0
   end
 end
 

data/lib/td/command/sched.rb

@@ -41,7 +41,13 @@ module Command
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
     }
-    op.on('-t', '--timezone TZ', 'name of the timezone (like Asia/Tokyo)') {|s|
+    op.on('-t', '--timezone TZ', "name of the timezone.",
+                                 "  Only extended timezones like 'Asia/Tokyo', 'America/Los_Angeles' are supported,",
+                                 "  (no 'PST', 'PDT', etc...).",
+                                 "  When a timezone is specified, the cron schedule is referred to that timezone.",
+                                 "  Otherwise, the cron schedule is referred to the UTC timezone.",
+                                 "  E.g. cron schedule '0 12 * * *' will execute daily at 5 AM without timezone option",
+                                 "  and at 12PM with the -t / --timezone 'America/Los_Angeles' timezone option") {|s|
       timezone = s
     }
     op.on('-D', '--delay SECONDS', 'delay time of the schedule', Integer) {|i|
@@ -97,7 +103,7 @@ module Command
 
     client = get_client
 
-    # local existance check
+    # local existence check
     get_database(client, db_name)
 
     begin
@@ -153,7 +159,13 @@ module Command
     op.on('-r', '--result RESULT_TABLE', 'change the result table') {|s|
       result = s
     }
-    op.on('-t', '--timezone TZ', 'change the name of the timezone (like Asia/Tokyo)') {|s|
+    op.on('-t', '--timezone TZ', "name of the timezone.",
+                                 "  Only extended timezones like 'Asia/Tokyo', 'America/Los_Angeles' are supported,",
+                                 "  (no 'PST', 'PDT', etc...).",
+                                 "  When a timezone is specified, the cron schedule is referred to that timezone.",
+                                 "  Otherwise, the cron schedule is referred to the UTC timezone.",
+                                 "  E.g. cron schedule '0 12 * * *' will execute daily at 5 AM without timezone option",
+                                 "  and at 12PM with the -t / --timezone 'America/Los_Angeles' timezone option") {|s|
       timezone = s
     }
     op.on('-D', '--delay SECONDS', 'change the delay time of the schedule', Integer) {|i|

data/lib/td/command/schema.rb

@@ -74,7 +74,11 @@ module Command
       name = name.to_s
       type = type.to_s
 
-      API.validate_column_name(name)
+      begin
+        API.validate_column_name(name)
+      rescue ParameterValidationError => e
+        raise ParameterConfigurationError, e
+      end
       #type = API.normalize_type_name(type)
 
       if schema.fields.find {|f| f.name == name }

data/lib/td/command/status.rb

@@ -34,7 +34,7 @@ module Command
     j = client.jobs(0, 4)
     j.each {|job|
       start = job.start_at
-      elapsed = cmd_format_elapsed(start, job.end_at)
+      elapsed = humanize_elapsed_time(start, job.end_at)
       jobs << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.result_url}
     }
     x2, y2 = status_render(0, 0, "[Jobs]", jobs, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query])

data/lib/td/command/table.rb

@@ -144,20 +144,27 @@ module Command
       databases = client.databases
     end
 
+    has_item = databases.select {|db| db.tables.select {|table| table.type == :item}.length > 0 }.length > 0
+
     rows = []
     ::Parallel.each(databases, :in_threads => num_threads) {|db|
       begin
+        db.tables.each {}
         db.tables.each {|table|
           pschema = table.schema.fields.map {|f|
             "#{f.name}:#{f.type}"
           }.join(', ')
-          rows << {
+          new_row = {
             :Database => db.name, :Table => table.name, :Type => table.type.to_s, :Count => TreasureData::Helpers.format_with_delimiter(table.count),
             :Size => show_size_in_bytes ? TreasureData::Helpers.format_with_delimiter(table.estimated_storage_size) : table.estimated_storage_size_string,
             'Last import' => table.last_import ? table.last_import.localtime : nil,
             'Last log timestamp' => table.last_log_timestamp ? table.last_log_timestamp.localtime : nil,
             :Schema => pschema
           }
+          if has_item and table.type == :item
+            new_row['Primary key'] = "#{table.primary_key}:#{table.primary_key_type}"
+          end
+          rows << new_row
         }
       rescue APIError => e
         # ignores permission error because db:list shows all databases
@@ -171,7 +178,13 @@ module Command
       [map[:Database], map[:Type].size, map[:Table]]
     }
 
-    puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Size, 'Last import', 'Last log timestamp', :Schema], :max_width=>500, :render_format => op.render_format)
+    fields = []
+    if has_item
+      fields = [:Database, :Table, :Type, :Count, :Size, 'Last import', 'Last log timestamp', 'Primary key', :Schema]
+    else
+      fields = [:Database, :Table, :Type, :Count, :Size, 'Last import', 'Last log timestamp', :Schema]
+    end
+    puts cmd_render_table(rows, :fields => fields, :max_width => 500, :render_format => op.render_format)
 
     if rows.empty?
       if db_name
@@ -207,10 +220,12 @@ module Command
 
     table = get_table(client, db_name, table_name)
 
-    puts "Name      : #{table.db_name}.#{table.name}"
-    puts "Type      : #{table.type}"
-    puts "Count     : #{table.count}"
-    puts "Schema    : ("
+    puts "Name        : #{table.db_name}.#{table.name}"
+    puts "Type        : #{table.type}"
+    puts "Count       : #{table.count}"
+    # p table.methods.each {|m| puts m}
+    puts "Primary key : #{table.primary_key}:#{table.primary_key_type}" if table.type == :item
+    puts "Schema      : ("
     table.schema.fields.each {|f|
       puts "    #{f.name}:#{f.type}"
     }
@@ -334,7 +349,7 @@ module Command
     to = nil
     wait = false
 
-    op.on('-t', '--to TIME', 'end time of logs to delete') {|s|
+    op.on('-t', '--to TIME', 'end time of logs to delete in Unix time multiple of 3600 (1 hour)') {|s|
       if s.to_i.to_s == s
         # UNIX time
         to = s.to_i
@@ -343,7 +358,7 @@ module Command
         to = Time.parse(s).to_i
       end
     }
-    op.on('-f', '--from TIME', 'start time of logs to delete') {|s|
+    op.on('-f', '--from TIME', 'start time of logs to delete in Unix time multiple of 3600 (1 hour)') {|s|
       if s.to_i.to_s == s
         from = s.to_i
       else
@@ -351,7 +366,7 @@ module Command
         from = Time.parse(s).to_i
       end
     }
-    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
+    op.on('-w', '--wait', 'wait for the job to finish', TrueClass) {|b|
       wait = b
     }
 
@@ -368,7 +383,7 @@ module Command
     end
 
     if from % 3600 != 0 || to % 3600 != 0
-      $stderr.puts "time must be a multiple of 3600 (1 hour)"
+      $stderr.puts "Time for the -f / --from and -t / --to options must be a multiple of 3600 (1 hour)"
       exit 1
     end
 
@@ -402,7 +417,7 @@ module Command
 
     $stderr.puts "Table set to expire data older than #{expire_days} days."
   end
-  
+
 
   IMPORT_TEMPLATES = {
     'apache' => [

data/lib/td/command/update.rb

@@ -4,21 +4,20 @@ module TreasureData
 module Command
 
   def update(op)
-    if TreasureData::Updater.disable?
-      $stderr.puts TreasureData::Updater.disable_message
+    # for gem installation, this command is disallowed -
+    #   it only works for the toolbelt.
+    if Updater.disable?
+      $stderr.puts Updater.disable_message
       exit
     end
 
-    $stderr.puts <<EOS
-Updating started at #{Time.now}
-from #{TreasureData::VERSION}
-EOS
-    if new_version = TreasureData::Updater.update
-      $stderr.puts "updated to #{new_version}"
+    start_time = Time.now
+    puts "Updating 'td' from #{TOOLBELT_VERSION}..."
+    if new_version = Updater.update
+      puts "Successfully updated to #{new_version} in #{humanize_time((Time.now - start_time).to_i)}."
     else
-      $stderr.puts "nothing to update"
+      puts "Nothing to update."
     end
-    $stderr.puts "ended at #{Time.now}"
   end
 
 end