td 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc5f6252aeace112a6886530da039c0761077ac0
-  data.tar.gz: 1dfddbf8f907474702693cd7be2a23fa940d0f14
+  metadata.gz: f7b9e3d57fd08024e999160fe1678031a293941e
+  data.tar.gz: 39454d414e3afddb9e83fcf3e3c574c98911536f
 SHA512:
-  metadata.gz: a6212b8724a6c971d5b75437c1eb540f3d72a9afddeefc23ddcea00fcedb6bc1b5a4846f6560412027eea4ae0ab7bdcef4467beab81c9395985dcbf217c96263
-  data.tar.gz: 65523ca21f55cdf2e852a6993020d61c5a55ea14e8a4b46b6c1280067960558be7b89b5fd3c22d2568ed63cc78e3e3445e49d7318d94330e6bebeef864f5c9e3
+  metadata.gz: d4846a9e5e75112f0350cbb4824e5335cb2861fabc927ddcacefe81a9a4894af08da0903920db5d1b4fcfd170b571341e5b41a77293c7a572df92ca4eec487e1
+  data.tar.gz: 0909c9942d35536d644db19f3ad0c3bf219d78ad59388ef8cee92af5c7f0d3271c4779503d5268dda0bd65cc2bcbaf231c3d5f02369b43dbcef0af50f77c364b

data/.travis.yml

@@ -16,3 +16,8 @@ script: bundle exec rake spec
 matrix:
   allow_failures:
     - rvm: ruby-head
+
+sudo: false
+
+notifications:
+  webhooks: http://td-beda.herokuapp.com/travisci_callback

data/ChangeLog

@@ -1,3 +1,18 @@
+== 2015-10-29 version 0.13.0
+
+* fix command error when recive 40X errror from server
+* support show config_diff in connector:[list,show,create,delete]
+* add new command `table:rename`. This command can rename table.
+* fix connector:history crash if history has queueing state.
+* fix connector:run time argument bug.
+* support query, job:show write Hash format JSON. If you use it, you set --column-header and -f json options.
+* change default file name create by connector:guess rename to `config.yml`.
+* remove --exclude option from connector:issue
+* support --g option for connector:config. It can specified guess plugin.
+* improve progressbar
+* support filter section for config file.
+* remove --primary-key option from table:create
+
 == 2015-08-10 version 0.12.0
 
 * Improve error message from APIError

data/lib/td/command/common.rb

@@ -247,6 +247,13 @@ EOS
     table
   end
 
+  def table_exist?(database, table_name)
+    database.table(table_name)
+    true
+  rescue
+    false
+  end
+
   def create_database_and_table_if_not_exist(client, db_name, table_name)
     # Merge with db_create and table_create after refactoring
     API.validate_database_name(db_name)
@@ -295,6 +302,22 @@ EOS
     end
   end
 
+  def puts_with_indent(message, indent_size = 4, io = $stdout)
+    io.puts "#{' ' * indent_size}#{message}"
+  end
+
+  def create_file_backup(out)
+    return unless File.exist?(out)
+    0.upto(100) do |idx|
+      backup = "#{out}.#{idx}"
+      unless File.exist?(backup)
+        FileUtils.mv(out, backup)
+        return
+      end
+    end
+    raise "backup file creation failed"
+  end
+
   def self.validate_api_endpoint(endpoint)
     require 'uri'
 
@@ -351,26 +374,28 @@ EOS
     files
   end
 
-  class DownloadProgressIndicator
-    def initialize(msg)
-      @base_msg = msg
-      $stdout.print @base_msg + " " * 10
-    end
-  end
-
-  class TimeBasedDownloadProgressIndicator < DownloadProgressIndicator
+  class TimeBasedDownloadProgressIndicator
     def initialize(msg, start_time, periodicity = 2)
+      require 'ruby-progressbar'
+
       @start_time = start_time
       @last_time = start_time
       @periodicity = periodicity
-      super(msg)
+
+      @progress_bar = ProgressBar.create(
+        title: msg,
+        total: nil,
+        format: formated_with_elasped_time(0),
+        output: $stdout,
+        unknown_progress_animation_steps: [' '],
+      )
+
+      update_progress_bar(formated_with_elasped_time(Command.humanize_elapsed_time(@start_time, @start_time)))
     end
 
     def update
       if (time = Time.now.to_i) - @last_time >= @periodicity
-        msg = "\r#{@base_msg}: #{Command.humanize_elapsed_time(@start_time, time)} elapsed"
-        $stdout.print "\r" + " " * (msg.length + 10)
-        $stdout.print msg
+        update_progress_bar(formated_with_elasped_time(Command.humanize_elapsed_time(@start_time, time)))
         @last_time = time
         true
       else
@@ -379,13 +404,37 @@ EOS
     end
 
     def finish
-      $stdout.puts "\r#{@base_msg}...done" + " " * 20
+      # NOTE %B is for clear terminal line
+      update_progress_bar("%t: done %B")
+    end
+
+    private
+
+    def formated_with_elasped_time(elasped_time)
+      # NOTE %B is for clear terminal line
+      "%t: #{elasped_time} elapsed %B"
+    end
+
+    def update_progress_bar(format)
+      @progress_bar.format = format
+      @progress_bar.refresh
     end
   end
 
-  class SizeBasedDownloadProgressIndicator < DownloadProgressIndicator
-    def initialize(msg, size, perc_step = 1, min_periodicity = nil)
-      @size = size
+  class SizeBasedDownloadProgressIndicator
+    def initialize(msg, total_size, perc_step = 1, min_periodicity = nil)
+      require 'ruby-progressbar'
+
+      @total_size = total_size
+      @total_byte_size = Command.humanize_bytesize(@total_size) unless unknown_progress_mode?
+
+      @progress_bar = ProgressBar.create(
+        title: msg,
+        total: unknown_progress_mode? ? nil : @total_size,
+        format: formated_title(0),
+        output: $stdout,
+      )
+      @progress_bar.refresh
 
       # perc_step is how small a percentage increment can be shown
       @perc_step = perc_step
@@ -397,22 +446,18 @@ EOS
       # track progress
       @last_perc_step = 0
       @last_time = @start_time
-
-      super(msg)
     end
 
     def update(curr_size)
-      if @size.nil? || @size == 0
-        msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)}"
-        $stdout.print msg
+      if unknown_progress_mode?
+        update_progress_bar(curr_size)
         true
       else
-        ratio = (curr_size.to_f * 100 / @size).round(1)
+        ratio = (curr_size.to_f * 100 / @total_size).round(1)
         if ratio >= (@last_perc_step + @perc_step) &&
            (!@min_periodicity || (time = Time.now.to_i) - @last_time >= @min_periodicity)
-          msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)} / #{ratio}%"
-          $stdout.print "\r" + " " * (msg.length + 10)
-          $stdout.print msg
+          update_progress_bar(curr_size)
+
           @last_perc_step = ratio
           @last_time = time
           true
@@ -423,7 +468,32 @@ EOS
     end
 
     def finish
-      $stdout.print "\r#{@base_msg}...done" + " " * 20
+      if unknown_progress_mode?
+        @progress_bar.format = "%t : #{Command.humanize_bytesize(@progress_bar.progress).rjust(10)} Done"
+        @progress_bar.progress = 0
+      else
+        update_progress_bar(@progress_bar.total)
+      end
+    end
+
+    private
+
+    def unknown_progress_mode?
+      @total_size.nil? || @total_size == 0
+    end
+
+    def update_progress_bar(curr_size)
+      @progress_bar.format = formated_title(curr_size)
+      @progress_bar.progress = curr_size
+    end
+
+    def formated_title(curr_size)
+      if unknown_progress_mode?
+        "%t : #{Command.humanize_bytesize(curr_size).rjust(10)}"
+      else
+        rjust_size = @total_byte_size.size + 1
+        "%t #{Command.humanize_bytesize(curr_size).rjust(rjust_size)} / #{@total_byte_size.rjust(rjust_size)} : %w "
+      end
     end
   end
 

data/lib/td/command/connector.rb

@@ -1,8 +1,10 @@
 require 'td/command/common'
 require 'td/command/job'
+require 'td/connector_config_normalizer'
 require 'json'
 require 'uri'
 require 'yaml'
+require 'time'
 
 module TreasureData
 module Command
@@ -16,13 +18,17 @@ module Command
   def connector_guess(op)
     type = 's3'
     id = secret = source = nil
-    out = 'td-bulkload.yml'
+    out = 'config.yml'
+    guess_plugins = {}
 
     op.on('--type[=TYPE]', "(obsoleted)") { |s| type = s }
     op.on('--access-id ID', "(obsoleted)") { |s| id = s }
     op.on('--access-secret SECRET', "(obsoleted)") { |s| secret = s }
     op.on('--source SOURCE', "(obsoleted)") { |s| source = s }
     op.on('-o', '--out FILE_NAME', "output file name for connector:preview") { |s| out = s }
+    op.on('-g', '--guess NAME,NAME,...', 'specify list of guess plugins that users want to use') {|s|
+      guess_plugins['guess_plugins'] = s.split(',')
+    }
 
     config_file = op.cmd_parse
     if config_file
@@ -30,6 +36,7 @@ module Command
       out ||= config_file
     else
       begin
+        $stdout.puts 'Command line option is obsoleted. You should use configuration file.'
         required('--access-id', id)
         required('--access-secret', secret)
         required('--source', source)
@@ -60,14 +67,17 @@ module Command
       }
     end
 
+    config = TreasureData::ConnectorConfigNormalizer.new(config).normalized_config
+    config['exec'].merge!(guess_plugins)
+
     client = get_client
     job = client.bulk_load_guess(config: config)
 
-    create_bulkload_job_file_backup(out)
+    create_file_backup(out)
     if /\.json\z/ =~ out
       config_str = JSON.pretty_generate(job['config'])
     else
-      config_str = YAML.dump(job['config'])
+      config_str = config_to_yaml(job['config'])
     end
     File.open(out, 'w') do |f|
       f << config_str
@@ -110,14 +120,13 @@ module Command
   def connector_issue(op)
     database = table = nil
     time_column      = nil
-    wait = exclude   = false
+    wait             = false
     auto_create      = false
 
     op.on('--database DB_NAME', "destination database") { |s| database = s }
     op.on('--table TABLE_NAME', "destination table") { |s| table = s }
     op.on('--time-column COLUMN_NAME', "data partitioning key") { |s| time_column = s }  # unnecessary but for backward compatibility
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) { |b| wait = b }
-    op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) { |b| exclude = b }
     op.on('--auto-create-table', "Create table and database if doesn't exist", TrueClass) { |b|
       auto_create = b
     }
@@ -142,7 +151,7 @@ module Command
     $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job_id}' to show the status."
 
     if wait
-      wait_connector_job(client, job_id, exclude)
+      wait_connector_job(client, job_id)
     end
   end
 
@@ -152,7 +161,7 @@ module Command
 
     client = get_client()
     # TODO database and table is empty at present. Fix API or Client.
-    keys = ['name', 'cron', 'timezone', 'delay', 'database', 'table', 'config']
+    keys = ['name', 'cron', 'timezone', 'delay', 'database', 'table', 'config', 'config_diff']
     fields = keys.map { |e| e.capitalize.to_sym }
     rows = client.bulk_load_list().sort_by { |e|
       e['name']
@@ -229,35 +238,47 @@ module Command
     fields = [:JobID, :Status, :Records, :Database, :Table, :Priority, :Started, :Duration]
     client = get_client()
     rows = client.bulk_load_history(name).map { |e|
+      time_property = if e['start_at']
+        {
+          :Started => Time.at(e['start_at']),
+          :Duration => (e['end_at'].nil? ? Time.now.to_i : e['end_at']) - e['start_at'],
+        }
+      else
+        {:Started => '', :Duration => ''}
+      end
+
       {
         :JobID => e['job_id'],
         :Status => e['status'],
         :Records => e['records'],
         # TODO: td-client-ruby should retuan only name
-        :Database => e['database']['name'],
-        :Table => e['table']['name'],
+        :Database => e['database'] ? e['database']['name'] : '',
+        :Table    => e['table']    ? e['table']['name']    : '',
         :Priority => e['priority'],
-        :Started => Time.at(e['start_at']),
-        :Duration => (e['end_at'].nil? ? Time.now.to_i : e['end_at']) - e['start_at'],
-      }
+      }.merge(time_property)
     }
     $stdout.puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format)
   end
 
   def connector_run(op)
-    wait = exclude = false
+    wait = false
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) { |b| wait = b }
-    op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) { |b| exclude = b }
 
     name, scheduled_time = op.cmd_parse
+    time = if scheduled_time
+      Time.parse(scheduled_time).to_i
+    else
+      current_time.to_i
+    end
 
     client = get_client()
-    job_id = client.bulk_load_run(name)
+    job_id = client.bulk_load_run(name, time)
+
     $stdout.puts "Job #{job_id} is queued."
     $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job_id}' to show the status."
 
     if wait
-      wait_connector_job(client, job_id, exclude)
+      wait_connector_job(client, job_id)
     end
   end
 
@@ -277,6 +298,19 @@ private
     nil
   end
 
+  def config_to_yaml(config)
+    config_str = ''
+    begin
+      require 'td/compact_format_yamler'
+      config_str = TreasureData::CompactFormatYamler.dump(config)
+    rescue
+      # NOTE fail back
+      config_str = YAML.dump(config)
+    end
+    config_str
+  end
+
+
   def prepare_bulkload_job_config(config_file)
     unless File.exist?(config_file)
       raise ParameterConfigurationError, "configuration file: #{config_file} not found"
@@ -293,25 +327,7 @@ private
       raise ParameterConfigurationError, "configuration file: #{config_file} #{e.message}"
     end
 
-    if config['config']
-      if config.size != 1
-        raise "Setting #{(config.keys - ['config']).inspect} keys in a configuration file is not supported. Please set options to the command line argument."
-      end
-      config = config['config']
-    end
-    config
-  end
-
-  def create_bulkload_job_file_backup(out)
-    return unless File.exist?(out)
-    0.upto(100) do |idx|
-      backup = "#{out}.#{idx}"
-      unless File.exist?(backup)
-        FileUtils.mv(out, backup)
-        return
-      end
-    end
-    raise "backup file creation failed"
+    TreasureData::ConnectorConfigNormalizer.new(config).normalized_config
   end
 
   def dump_connector_session(session)
@@ -323,13 +339,19 @@ private
     $stdout.puts "Table    : #{session["table"]}"
     $stdout.puts "Config"
     $stdout.puts YAML.dump(session["config"])
+    $stdout.puts
+    $stdout.puts "Config Diff"
+    $stdout.puts YAML.dump(session["config_diff"])
   end
 
-  def wait_connector_job(client, job_id, exclude)
+  def wait_connector_job(client, job_id)
     job = client.job(job_id)
     wait_job(job, true)
     $stdout.puts "Status     : #{job.status}"
   end
 
+  def current_time
+    Time.now
+  end
 end
 end

data/lib/td/command/import.rb

@@ -1,5 +1,6 @@
 require 'td/updater'
 require 'time'
+require 'yaml'
 
 module TreasureData
 module Command
@@ -77,6 +78,50 @@ module Command
     bulk_import_unfreeze(op)
   end
 
+  def import_config(op)
+    out = 'td-bulkload.yml'
+    options = {
+      'format' => 'csv'
+    }
+    not_migrate_options = []
+    op.on('-o', '--out FILE_NAME', "output file name for connector:guess") { |s| out = s }
+    op.on('-f', '--format FORMAT', "source file format [csv, tsv, mysql]; default=csv") { |s| options['format'] = s }
+
+    op.on('--db-url URL',           "Database Connection URL") { |s| options['db_url']      = s }
+    op.on('--db-user NAME',         "user name for database")  { |s| options['db_user']     = s }
+    op.on('--db-password PASSWORD', "password for database")   { |s| options['db_password'] = s }
+    %w(--columns --column-header --time-column --time-format).each do |not_migrate_option|
+      opt_arg_name = not_migrate_option.gsub('--', '').upcase
+      op.on("#{not_migrate_option} #{opt_arg_name}", 'not supported') { |s| not_migrate_options << not_migrate_option }
+    end
+
+    arg = op.cmd_parse
+
+    unless %w(mysql csv tsv).include?(options['format'])
+      raise ParameterConfigurationError, "#{options['format']} is unknown format. Support format is csv, tsv and mysql."
+    end
+
+    unless not_migrate_options.empty?
+      be = not_migrate_options.size == 1 ? 'is' : 'are'
+      $stderr.puts "`#{not_migrate_options.join(', ')}` #{be} not migrate. Please, edit config file after execute guess commands."
+    end
+
+    $stdout.puts "Generating #{out}..."
+
+    config = generate_seed_confing(options['format'], arg, options)
+    config_str = YAML.dump(config)
+
+
+    create_file_backup(out)
+    File.open(out, 'w') {|f| f << config_str }
+
+    if config['out']['type'] == 'td'
+      show_message_for_td_output_plugin(out)
+    else
+      show_message_for_td_data_connector(out)
+    end
+  end
+
   #
   # Module private methods - don't map to import:* commands
   #
@@ -270,7 +315,7 @@ module Command
           port = 80 if port == 0
           ssl = false
         end
-        end
+      end
 
       sysprops << "-Dtd.api.server.scheme=#{ssl ? 'https' : 'http'}://"
       sysprops << "-Dtd.api.server.host=#{host}"
@@ -344,5 +389,121 @@ module Command
     version.first
   end
 
+  def generate_seed_confing(format, arg, options)
+    case format
+    when 'csv', 'tsv'
+      if arg =~ /^s3:/
+        generate_s3_config(format, arg)
+      else
+        generate_csv_config(format, arg)
+      end
+    when 'mysql'
+      arg = arg[1] unless arg.class == String
+      generate_mysql_config(arg, options)
+    else
+      # NOOP
+    end
+  end
+
+  def generate_s3_config(format, arg)
+    puts_with_indent('Using S3 input')
+    puts_with_indent('Using CSV parser plugin')
+    puts_with_indent('Using Treasure Data data connector')
+
+    match = Regexp.new("^s3://(.*):(.*)@/([^/]*)/(.*)").match(arg)
+
+    {
+      'in' => {
+        'type' => 's3',
+        'access_key_id'     => match[1],
+        'secret_access_key' => match[2],
+        'bucket'            => match[3],
+        'path_prefix'       => normalize_path_prefix(match[4])
+      },
+      'out' => {'mode' => 'append'}
+    }
+  end
+
+  def generate_csv_config(format, arg)
+    puts_with_indent('Using local file input')
+    puts_with_indent('Using CSV parser plugin')
+    puts_with_indent('Using Treasure Data output')
+
+    {
+      'in' => {
+        'type'        => 'file',
+        'path_prefix' => normalize_path_prefix(arg),
+        'decorders'   => [{'type' => 'gzip'}],
+      },
+      'out' => td_output_config,
+    }
+  end
+
+  def td_output_config
+    {
+      'type' => 'td',
+      'endpoint' => Config.cl_endpoint || Config.endpoint,
+      'apikey' => Config.cl_apikey || Config.apikey,
+      'database' => '',
+      'table' => '',
+    }
+  end
+
+  def normalize_path_prefix(path)
+    path.gsub(/\*.*/, '')
+  end
+
+  def generate_mysql_config(arg, options)
+    puts_with_indent('Using MySQL input')
+    puts_with_indent('Using MySQL parser plugin')
+    puts_with_indent('Using Treasure Data output')
+
+    mysql_url_regexp = Regexp.new("[jdbc:]*mysql://(?<host>[^:/]*)[:]*(?<port>[^/]*)/(?<db_name>.*)")
+
+    config = if (match = mysql_url_regexp.match(options['db_url']))
+      {
+        'host'     => match['host'],
+        'port'     => match['port'] == '' ? 3306 : match['port'].to_i,
+        'database' => match['db_name'],
+      }
+    else
+      {
+        'host'     => '',
+        'port'     => 3306,
+        'database' => '',
+      }
+    end
+
+    {
+      'in' => config.merge(
+        'type'     => 'mysql',
+        'user'     => options['db_user'],
+        'password' => options['db_password'],
+        'table'    => arg,
+        'select'   => '*',
+      ),
+      'out' => td_output_config,
+    }
+  end
+
+    def show_message_for_td_output_plugin(out)
+      $stdout.puts 'Done. Please use embulk to load the files.'
+      $stdout.puts 'Next steps:'
+      $stdout.puts
+      puts_with_indent '# install embulk'
+      puts_with_indent "$ embulk gem install embulk-output-td"
+      puts_with_indent '$ embulk guess seed.yml -o config.yml'
+      puts_with_indent '$ embulk preview config.yml'
+      puts_with_indent '$ embulk run config.yml'
+    end
+
+    def show_message_for_td_data_connector(out)
+      $stdout.puts 'Done. Please use connector:guess and connector:run to load the files.'
+      $stdout.puts 'Next steps:'
+      puts_with_indent "$ td connector:guess #{out} -o config.yml"
+      puts_with_indent '$ td connector:preview config.yml'
+      puts_with_indent '$ td connector:run config.yml'
+    end
+
 end
 end