td 0.12.0 → 0.13.0

data/lib/td/command/job.rb

@@ -122,9 +122,9 @@ module Command
     end
 
     if render_opts[:header]
-      unless ['tsv', 'csv'].include?(format)
+      unless ['json', 'tsv', 'csv'].include?(format)
         raise ParameterConfigurationError,
-              "Option -c / --column-header is only supported with tsv and csv formats"
+              "Option -c / --column-header is only supported with json, tsv and csv formats"
       end
     end
 
@@ -306,27 +306,15 @@ private
 
     case format
     when 'json'
-      require 'yajl'
-      open_file(tempfile || output, "w") {|f|
-        f.write "["
-        n_rows = 0
-        unless output.nil?
-          indicator = Command::SizeBasedDownloadProgressIndicator.new(
-            "NOTE: the job result is being written to #{output} in json format",
-            job.result_size, 0.1, 1)
-        end
-        job.result_each_with_compr_size {|row, compr_size|
-          indicator.update(compr_size) unless output.nil?
-          f.write ",\n" if n_rows > 0
-          f.write Yajl.dump(row)
-          n_rows += 1
-          break if output.nil? and !limit.nil? and n_rows == limit
-        }
-        f.write "]"
-        indicator.finish unless output.nil?
-      }
-      $stdout.puts if output.nil?
+      if render_opts[:header] && job.hive_result_schema
+        headers = job.hive_result_schema.map {|name, _| name }
 
+        write_result_for_json(job, output, tempfile, limit, render_opts) {|row|
+          Hash[headers.zip(row)]
+        }
+      else
+        write_result_for_json(job, output, tempfile, limit, render_opts) {|row| row }
+      end
     when 'csv'
       require 'yajl'
       require 'csv'
@@ -444,6 +432,29 @@ private
     end
   end
 
+  def write_result_for_json(job, output, tempfile, limit, render_opts)
+    require 'yajl'
+    open_file(tempfile || output, "w") {|f|
+      f.write "["
+      n_rows = 0
+      unless output.nil?
+        indicator = Command::SizeBasedDownloadProgressIndicator.new(
+          "NOTE: the job result is being written to #{output} in json format",
+          job.result_size, 0.1, 1)
+      end
+      job.result_each_with_compr_size {|row, compr_size|
+        indicator.update(compr_size) unless output.nil?
+        f.write ",\n" if n_rows > 0
+        f.write Yajl.dump(yield(row))
+        n_rows += 1
+        break if output.nil? and !limit.nil? and n_rows == limit
+      }
+      f.write "]"
+      indicator.finish unless output.nil?
+    }
+    $stdout.puts if output.nil?
+  end
+
   def render_result(job, limit, format=nil, render_opts={})
     require 'yajl'
 
@@ -515,4 +526,3 @@ private
 
 end # module Command
 end # module TrasureData
-

data/lib/td/command/list.rb

@@ -236,6 +236,7 @@ module List
   add_list 'table:import', %w[db table files_], 'Parse and import files to a table', ['table:import example_db table1 --apache access.log', 'table:import example_db table1 --json -t time - < test.json']
   add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', ['table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY']
   add_list 'table:swap', %w[db table1 table2], 'Swap names of two tables', ['table:swap example_db table1 table2']
+  add_list 'table:rename', %w[db from_table dest_table], 'rename exist table', ['table:rename example_db table1 table2']
   add_list 'table:tail', %w[db table], 'Get recently imported logs', ['table:tail example_db table1', 'table:tail example_db table1 -n 30']
   add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', ['table:partial_delete example_db table1 --from 1341000000 --to 1341003600']
   add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days. Set to 0 to disable the expiration.', ['table:expire example_db table1 30']
@@ -269,6 +270,7 @@ module List
   add_list 'import:delete', %w[name], 'Delete a bulk import session', ['import:delete logs_201201']
   add_list 'import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', ['import:freeze logs_201201']
   add_list 'import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', ['import:unfreeze logs_201201']
+  add_list 'import:config', %w[files_], 'create guess config from arguments', ['import:config "s3://<s3_access_key>:<s3_secret_key>@/my_bucket/path/to/*.csv" -o seed.yml']
 
   add_list 'result:list', %w[], 'Show list of result URLs', ['result:list', 'results']
   add_list 'result:show', %w[name], 'Describe information of a result URL', ['result name']
@@ -353,7 +355,7 @@ module List
   add_list 'connector:update', %w[name config], 'Modify connector session', ['connector:update connector1 td-bulkload.yml']
   add_list 'connector:delete', %w[name], 'Delete connector session', ['connector:delete connector1']
   add_list 'connector:history', %w[name], 'Show job history of connector session', ['connector:history connector1']
-  add_list 'connector:run', %w[name time], 'Run connector session for the specified time', ['connector:run connector1 "2016-01-01 00:00:00"']
+  add_list 'connector:run', %w[name time?], 'Run connector with session for the specified time option', ['connector:run connector1 "2016-01-01 00:00:00"']
 
   # aliases
   add_alias 'db', 'db:show'
@@ -441,4 +443,3 @@ module List
 end # module List
 end # module Command
 end # module TreasureData
-

data/lib/td/command/query.rb

@@ -76,7 +76,7 @@ module Command
       end
       limit = s.to_i
     }
-    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to tsv and csv formats)', TrueClass) {|b|
+    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to json, tsv and csv formats)', TrueClass) {|b|
       render_opts[:header] = b
     }
     op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
@@ -115,9 +115,9 @@ module Command
     end
 
     if render_opts[:header]
-      unless ['tsv', 'csv'].include?(format)
+      unless ['tsv', 'csv', 'json'].include?(format)
         raise ParameterConfigurationError,
-              "Option -c / --column-header is only supported with tsv and csv formats"
+              "Option -c / --column-header is only supported with json, tsv and csv formats"
       end
     end
 
@@ -155,5 +155,3 @@ module Command
   require 'td/command/job'  # wait_job, job_priority_id_of
 end
 end
-
-

data/lib/td/command/runner.rb

@@ -190,7 +190,7 @@ EOF
         show_backtrace "Error #{$!.class}: backtrace:", $!.backtrace
       end
 
-      if $!.respond_to? :api_backtrace
+      if $!.respond_to?(:api_backtrace) && $!.api_backtrace
         show_backtrace "Error backtrace from server:", $!.api_backtrace.split("\n")
       end
 
@@ -227,4 +227,3 @@ end # class Runner
 
 end # module Command
 end # module TreasureData
-

data/lib/td/command/table.rb

@@ -15,8 +15,6 @@ module Command
 
   def table_create(op)
     type = nil
-    primary_key = nil
-    primary_key_type = nil
 
     op.on('-T', '--type TYPE', 'set table type (log)') {|s|
       unless s == 'log'
@@ -24,21 +22,6 @@ module Command
       end
       type = s.to_sym
     }
-    op.on('--primary-key PRIMARY_KEY_AND_TYPE', '[primary key]:[primary key type(int or string)]') {|s|
-      unless /\A[\w]+:(string|int)\z/ =~ s
-        $stderr.puts "--primary-key PRIMARY_KEY_AND_TYPE is required, and should be in the format [primary key]:[primary key type]"
-        exit 1
-      end
-
-      args = s.split(':')
-      if args.length != 2
-        # this really shouldn't happen with the above regex
-        exit 1
-      end
-      primary_key = args[0]
-      primary_key_type = args[1]
-    }
-
     db_name, table_name = op.cmd_parse
 
     API.validate_table_name(table_name)
@@ -204,6 +187,35 @@ module Command
     $stderr.puts "'#{db_name}.#{table_name1}' and '#{db_name}.#{table_name2}' are swapped."
   end
 
+  def table_rename(op)
+    overwrite = false
+    op.on('--overwrite', 'replace existing dest table') { overwrite = true }
+    database_name, from_table_name, dest_table_name = op.cmd_parse
+
+    client = get_client
+    database = get_database(client, database_name)
+
+    unless table_exist?(database, from_table_name)
+      raise ParameterConfigurationError, "From table `#{from_table_name}` isn't exist."
+    end
+
+    if table_exist?(database, dest_table_name)
+      unless overwrite
+        raise ParameterConfigurationError, "Dest table `#{dest_table_name}` is exist. If you want to overwrite dest table, you should set `overwrite` option."
+      end
+    else
+      client.create_log_table(database_name, dest_table_name)
+    end
+
+    client.swap_table(database_name, from_table_name, dest_table_name)
+    client.delete_table(database_name, from_table_name)
+
+    $stderr.puts "'renamed from '#{database_name}.#{from_table_name}' to '#{database_name}.#{dest_table_name}'."
+  rescue ParameterConfigurationError => e
+    $stderr.puts e.message
+    exit 1
+  end
+
   def table_show(op)
     db_name, table_name = op.cmd_parse
 
@@ -673,4 +685,3 @@ module Command
   require 'td/command/job'  # wait_job
 end
 end
-

data/lib/td/compact_format_yamler.rb

@@ -0,0 +1,41 @@
+require 'psych'
+
+module TreasureData
+  module CompactFormatYamler
+    module Visitors
+      class YAMLTree < Psych::Visitors::YAMLTree
+        # NOTE support 2.0 following
+        unless self.respond_to? :create
+          class << self
+            alias :create :new
+          end
+        end
+
+        def visit_Hash o
+          if o.class == ::Hash && o.values.all? {|v| v.kind_of?(Numeric) || v.kind_of?(String) || v.kind_of?(Symbol) }
+            register(o, @emitter.start_mapping(nil, nil, true, Psych::Nodes::Mapping::FLOW))
+
+            o.each do |k,v|
+              accept k
+              accept v
+            end
+            @emitter.end_mapping
+          else
+            super
+          end
+        end
+      end
+    end
+
+    def self.dump(o, io = nil, options = {})
+      if Hash === io
+        options = io
+        io = nil
+      end
+
+      visitor = ::TreasureData::CompactFormatYamler::Visitors::YAMLTree.create options
+      visitor << o
+      visitor.tree.yaml io, options
+    end
+  end
+end

data/lib/td/connector_config_normalizer.rb

@@ -0,0 +1,32 @@
+module TreasureData
+  class ConnectorConfigNormalizer
+    def initialize(config)
+      @config = config
+    end
+
+    def normalized_config
+      case
+      when @config['in']
+        {
+          'in'      => @config['in'],
+          'out'     => @config['out']  || {},
+          'exec'    => @config['exec'] || {},
+          'filters' => @config['filters'] || []
+        }
+      when @config['config']
+        if @config.size != 1
+          raise "Setting #{(@config.keys - ['config']).inspect} keys in a configuration file is not supported. Please set options to the command line argument."
+        end
+
+        self.class.new(@config['config']).normalized_config
+      else
+        {
+          'in'   => @config,
+          'out'  => {},
+          'exec' => {},
+          'filters' => []
+        }
+      end
+    end
+  end
+end

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.12.0'
+  TOOLBELT_VERSION = '0.13.0'
 end

data/spec/spec_helper.rb

@@ -49,3 +49,21 @@ def execute_td(command_line)
 end
 
 class CallSystemExitError < RuntimeError; end
+
+shared_context 'quiet_out' do
+  let(:stdout_io) { StringIO.new }
+  let(:stderr_io) { StringIO.new }
+
+  around do |example|
+    out = $stdout.dup
+    err= $stdout.dup
+    begin
+      $stdout = stdout_io
+      $stderr = stderr_io
+      example.call
+    ensure
+      $stdout = out
+      $stderr = err
+    end
+  end
+end

data/spec/td/command/connector_spec.rb

@@ -5,6 +5,49 @@ require 'td/command/connector'
 
 module TreasureData::Command
   describe 'connector commands' do
+    describe '#connector_guess' do
+      let :command do
+        Class.new { include TreasureData::Command }.new
+      end
+
+      describe 'guess plugins' do
+        let(:guess_plugins) { %w(json query_string) }
+
+        let(:in_file)  { Tempfile.new('in.yml').tap{|f| f.close } }
+        let(:out_file) { Tempfile.new('out.yml').tap{|f| f.close } }
+
+        let(:option) {
+          List::CommandParser.new("connector:guess", ["config"], [], nil, [in_file.path, '-o', out_file.path, '--guess', guess_plugins.join(',')], true)
+        }
+        let(:client) { double(:client) }
+
+        before do
+          command.stub(:get_client).and_return(client)
+        end
+
+        let(:config) {
+          {
+            'in' => {'type' => 's3'}
+          }
+        }
+        let(:expect_config) {
+          config.merge('out' => {}, 'exec' => {'guess_plugins' => guess_plugins}, 'filters' => [])
+        }
+
+        include_context 'quiet_out'
+
+        before do
+          command.stub(:prepare_bulkload_job_config).and_return(config)
+        end
+
+        it 'guess_plugins passed td-client' do
+          client.should_receive(:bulk_load_guess).with({config: expect_config}).and_return({})
+
+          command.connector_guess(option)
+        end
+      end
+    end
+
     describe '#connector_preview' do
       let :command do
         Class.new { include TreasureData::Command }.new
@@ -129,5 +172,115 @@ module TreasureData::Command
         end
       end
     end
+
+    describe '#connector_run' do
+      include_context 'quiet_out'
+
+      let :command do
+        Class.new { include TreasureData::Command }.new
+      end
+      let(:client) { double(:client) }
+      let(:job_name) { 'job_1' }
+
+      before do
+        command.stub(:get_client).and_return(client)
+        client.stub(:database)
+      end
+
+      context 'with scheduled_time' do
+        let(:scheduled_time) { Time.now + 60 }
+        let(:option) {
+          List::CommandParser.new('connector:run', ['name'], ['time'], nil, [job_name, scheduled_time.strftime("%Y-%m-%d %H:%M:%S")], true)
+        }
+
+        it 'client call with unix time' do
+          client.should_receive(:bulk_load_run).with(job_name, scheduled_time.to_i).and_return(123)
+
+          command.connector_run(option)
+        end
+      end
+
+      context 'without scheduled_time' do
+        let(:option) {
+          List::CommandParser.new('connector:run', ['name'], ['time'], nil, [job_name], true)
+        }
+        let(:current_time) { Time.now }
+
+        it 'client call with unix time' do
+          client.should_receive(:bulk_load_run).with(job_name, current_time.to_i).and_return(123)
+          command.stub(:current_time).and_return(current_time.to_i)
+
+          command.connector_run(option)
+        end
+      end
+    end
+
+    describe 'connector history' do
+      include_context 'quiet_out'
+
+      let :command do
+        Class.new { include TreasureData::Command }.new
+      end
+      let(:name) { 'connector_test' }
+
+      subject do
+        op = List::CommandParser.new("connector:history", ["name"], [], nil, [name], true)
+        command.connector_history(op)
+      end
+
+      before do
+        client = double(:client)
+        client.stub(:bulk_load_history).with(name).and_return(history)
+        command.stub(:get_client).and_return(client)
+      end
+
+      context 'history is empty' do
+        let(:history) { [] }
+
+        it { expect { subject }.not_to raise_error }
+      end
+
+      context 'history in not empty' do
+        let(:history) { [column] }
+        let(:column) {
+          # TODO set real value
+          {
+            'job_id'   => '',
+            'status'   => '',
+            'records'  => '',
+            'database' => {'name' => ''},
+            'table'    => {'name' => ''},
+            'priority' =>  ''
+          }
+        }
+
+        context 'job is queueing' do
+          before do
+            column['start_at'] = nil
+            column['end_at']   = nil
+          end
+
+          it { expect { subject }.not_to raise_error }
+        end
+
+        context 'job is running' do
+          before do
+            column['start_at'] = Time.now.to_i
+            column['end_at']   = nil
+          end
+
+          it { expect { subject }.not_to raise_error }
+        end
+
+        context 'jobi is finished' do
+          before do
+            column['start_at'] = Time.now.to_i
+            column['end_at']   = (Time.now + 60).to_i
+          end
+
+          it { expect { subject }.not_to raise_error }
+        end
+      end
+    end
   end
 end