td 0.11.12 → 0.11.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36266b9c823ecc10de37fafc29ed07458a50a728
-  data.tar.gz: 92ec61eb2983e7b284565252729aaedadf1483f9
+  metadata.gz: 844dda6ec2ea3ce703d94332d12f07da0c8130c1
+  data.tar.gz: 2fe722894996e437589d8841b06dcf0d8da0534d
 SHA512:
-  metadata.gz: 545c05fd90e3577b175d732742182fca1bdd02dd9bf58666213006e89c0033e6ded3410cd610cc4f503b67db1f6e0f630d75249f33cd4f1ab56c326f0106b380
-  data.tar.gz: 92ff25f768e6a82dceb87fddb41e395624a2de9b0e8bd64160b584f709cf28bfaaa7790f8cd5bfc3fe2de7169cf3d7305d99024eaafc2afec7a2735d51eb70ae
+  metadata.gz: 467d5b84f145cb1a2e11926b8332018eed6734fab410bbae0271873b4481ea1755c2aea6475e5835cef7981fb3f18605e7a5b8fbe983f9a608fbd2ca447ae12c
+  data.tar.gz: c1fae992cf68fc41304fe1b8326e8d1782ca29ad0b6b52828bb75553f66df0b4aa85ebf33888a0516e04929bbcdd1a406abb6cd25f68b77a043741e1c79ccb3b

data/ChangeLog

@@ -1,3 +1,8 @@
+== 2015-07-29 version 0.11.13
+
+* allow retry when `td query -w` fail show result.
+* `connector:issue` create database and table if use `--auto-create-table`
+
 == 2015-07-27 version 0.11.12
 
 * query command support msgpack.gz format save

data/lib/td/command/common.rb

@@ -247,6 +247,30 @@ EOS
     table
   end
 
+  def create_database_and_table_if_not_exist(client, db_name, table_name)
+    # Merge with db_create and table_create after refactoring
+    API.validate_database_name(db_name)
+    begin
+      client.database(db_name)
+    rescue NotFoundError
+      begin
+        client.create_database(db_name)
+        $stderr.puts "Database '#{db_name}' is created."
+      rescue AlreadyExistsError
+        # do nothing
+      end
+    rescue ForbiddenError
+      # do nothing
+    end
+
+    API.validate_table_name(table_name)
+    begin
+      client.create_log_table(db_name, table_name)
+      $stderr.puts "Table '#{db_name}.#{table_name}' is created."
+    rescue AlreadyExistsError
+    end
+  end
+
   def ask_password(max=3, &block)
     3.times do
       begin

data/lib/td/command/connector.rb

@@ -109,13 +109,18 @@ module Command
 
   def connector_issue(op)
     database = table = nil
-    time_column = nil
-    wait = exclude = false
+    time_column      = nil
+    wait = exclude   = false
+    auto_create      = false
+
     op.on('--database DB_NAME', "destination database") { |s| database = s }
     op.on('--table TABLE_NAME', "destination table") { |s| table = s }
     op.on('--time-column COLUMN_NAME', "data partitioning key") { |s| time_column = s }  # unnecessary but for backward compatibility
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) { |b| wait = b }
     op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) { |b| exclude = b }
+    op.on('--auto-create-table', "Create table and database if doesn't exist", TrueClass) { |b|
+      auto_create = b
+    }
 
     config_file = op.cmd_parse
 
@@ -126,6 +131,11 @@ module Command
     (config['out'] ||= {})['time_column'] = time_column if time_column  # TODO will not work once embulk implements multi-job
 
     client = get_client()
+
+    if auto_create
+      create_database_and_table_if_not_exist(client, database, table)
+    end
+
     job_id = client.bulk_load_issue(database, table, config: config)
 
     $stdout.puts "Job #{job_id} is queued."

data/lib/td/command/query.rb

@@ -142,11 +142,10 @@ module Command
 
     if wait
       wait_job(job, true)
-      $stdout.puts "Status     : #{job.status}"
+      $stdout.puts "Status      : #{job.status}"
       if job.success? && !exclude
-        $stdout.puts "Result     :"
         begin
-          show_result(job, output, limit, format, render_opts)
+          show_result_with_retry(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e
         end
       end

data/lib/td/command/table.rb

@@ -372,28 +372,30 @@ module Command
     time_key = 'time'
     auto_create = false
 
+    import_params = {}
+
     op.on('--format FORMAT', "file format (default: #{format})") {|s|
-      format = s
+      import_params[:format] = s
     }
 
     op.on('--apache', "same as --format apache; apache common log format") {
-      format = 'apache'
+      import_params[:format] = 'apache'
     }
 
     op.on('--syslog', "same as --format syslog; syslog") {
-      format = 'syslog'
+      import_params[:format] = 'syslog'
     }
 
     op.on('--msgpack', "same as --format msgpack; msgpack stream format") {
-      format = 'msgpack'
+      import_params[:format] = 'msgpack'
     }
 
     op.on('--json', "same as --format json; LF-separated json format") {
-      format = 'json'
+      import_params[:format] = 'json'
     }
 
     op.on('-t', '--time-key COL_NAME', "time key name for json and msgpack format (e.g. 'created_at')") {|s|
-      time_key = s
+      import_params[:time_key] = s
     }
 
     op.on('--auto-create-table', "Create table and database if doesn't exist", TrueClass) { |b|
@@ -401,63 +403,51 @@ module Command
     }
 
     db_name, table_name, *paths = op.cmd_parse
+    import_params[:db_name] = db_name
+    import_params[:table_name] = table_name
+    import_params[:paths] = paths
 
     client = get_client
 
     if auto_create
-      # Merge with db_create and table_create after refactoring
-      API.validate_database_name(db_name)
-      begin
-        client.database(db_name)
-      rescue NotFoundError
-        begin
-          client.create_database(db_name)
-          $stderr.puts "Database '#{db_name}' is created."
-        rescue AlreadyExistsError
-          # do nothing
-        end
-      rescue ForbiddenError
-        # do nothing
-      end
-
-      API.validate_table_name(table_name)
-      begin
-        client.create_log_table(db_name, table_name)
-        $stderr.puts "Table '#{db_name}.#{table_name}' is created."
-      rescue AlreadyExistsError
-      end
+      create_database_and_table_if_not_exist(client, db_name, table_name)
     end
 
-    case format
+    do_table_import(client, import_params)
+  end
+
+  private
+  def do_table_import(client, import_params)
+    case import_params[:format]
     when 'json', 'msgpack'
       #unless time_key
       #  $stderr.puts "-t, --time-key COL_NAME (e.g. '-t created_at') parameter is required for #{format} format"
       #  exit 1
       #end
-      if format == 'json'
+      if import_params[:format] == 'json'
         require 'json'
         require 'time'
-        parser = JsonParser.new(time_key)
+        parser = JsonParser.new(import_params[:time_key])
       else
-        parser = MessagePackParser.new(time_key)
+        parser = MessagePackParser.new(import_params[:time_key])
       end
 
     else  # apache, syslog
-      regexp, names, time_format = IMPORT_TEMPLATES[format]
+      regexp, names, time_format = IMPORT_TEMPLATES[import_params[:format]]
       if !regexp || !names || !time_format
-        $stderr.puts "Unknown format '#{format}'"
+        $stderr.puts "Unknown format '#{import_params[:format]}'"
         exit 1
       end
       parser = TextParser.new(names, regexp, time_format)
     end
 
     begin
-      db = client.database(db_name)
+      db = client.database(import_params[:db_name])
     rescue ForbiddenError => e
       $stdout.puts "Warning: database and table validation skipped - #{e.message}"
     else
       begin
-        table = db.table(table_name)
+        table = db.table(import_params[:table_name])
       rescue ForbiddenError => e
         $stdout.puts "Warning: table validation skipped - #{e.message}"
       end
@@ -466,7 +456,7 @@ module Command
     require 'zlib'
 
     begin
-      files = paths.map {|path|
+      files = import_params[:paths].map {|path|
         if path == '-'
           $stdin
         elsif path =~ /\.gz$/
@@ -484,14 +474,13 @@ module Command
     require 'tempfile'
     #require 'thread'
 
-    files.zip(paths).each {|file, path|
-      import_log_file(file, path, client, db_name, table_name, parser)
+    files.zip(import_params[:paths]).each {|file, path|
+      import_log_file(file, path, client, import_params[:db_name], import_params[:table_name], parser)
     }
 
     $stdout.puts "done."
   end
 
-  private
   def import_log_file(file, path, client, db_name, table_name, parser)
     $stdout.puts "importing #{path}..."
 

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.11.12'
+  TOOLBELT_VERSION = '0.11.13'
 end

data/spec/td/command/connector_spec.rb

@@ -58,29 +58,75 @@ module TreasureData::Command
         Class.new { include TreasureData::Command }.new
       end
 
+      let(:stderr_io) do
+        StringIO.new
+      end
+
       subject do
         backup = $stdout.dup
+        stderr_backup = $stderr.dup
         buf = StringIO.new
 
         begin
           $stdout = buf
+          $stderr = stderr_io
 
-          op = List::CommandParser.new("connector:issue", ["config"], ['database', 'table'], nil, [File.join("spec", "td", "fixture", "bulk_load.yml"), '--database', 'database', '--table', 'table'], true)
-          command.connector_issue(op)
+          command.connector_issue(option)
 
           buf.string
         ensure
           $stdout = backup
+          $stderr = stderr_backup
         end
       end
 
-      before do
-        client = double(:client, bulk_load_issue: 1234)
-        command.stub(:get_client).and_return(client)
+      describe 'queueing job' do
+        let(:option) {
+          List::CommandParser.new("connector:issue", ["config"], ['database', 'table'], nil, [File.join("spec", "td", "fixture", "bulk_load.yml"), '--database', 'database', '--table', 'table'], true)
+        }
+
+        before do
+          client = double(:client, bulk_load_issue: 1234)
+          command.stub(:get_client).and_return(client)
+          command.stub(:create_database_and_table_if_not_exist)
+        end
+
+        it 'should include too_long_column_name without truncated' do
+          expect(subject).to include "Job 1234 is queued."
+        end
       end
 
-      it 'should include too_long_column_name without truncated' do
-        expect(subject).to include "Job 1234 is queued."
+      describe 'distination table' do
+        let(:client) { double(:client, bulk_load_issue: 1234) }
+
+        before do
+          command.stub(:get_client).and_return(client)
+          client.stub(:database)
+        end
+
+        context 'set auto crate table option' do
+          let(:option) {
+            List::CommandParser.new("connector:issue", ["config"], ['database', 'table'], nil, [File.join("spec", "td", "fixture", "bulk_load.yml"), '--database', 'database', '--table', 'table', '--auto-create-table'], true)
+          }
+
+          it 'call create_database_and_table_if_not_exist' do
+            command.should_receive(:create_database_and_table_if_not_exist)
+
+            subject
+          end
+        end
+
+        context 'not set auto crate table option' do
+          let(:option) {
+            List::CommandParser.new("connector:issue", ["config"], ['database', 'table'], nil, [File.join("spec", "td", "fixture", "bulk_load.yml"), '--database', 'database', '--table', 'table'], true)
+          }
+
+          it 'call create_database_and_table_if_not_exist' do
+            command.should_not_receive(:create_database_and_table_if_not_exist)
+
+            subject
+          end
+        end
       end
     end
   end

data/spec/td/command/table_spec.rb

@@ -179,5 +179,43 @@ module TreasureData::Command
         end
       end
     end
+
+    describe '#table_import' do
+      let(:db_name)    { 'database' }
+      let(:table_name) { 'table' }
+      let(:client)     { double('client') }
+      let(:command)    { Class.new { include TreasureData::Command }.new }
+
+      describe 'auto create table' do
+        before do
+          command.stub(:get_client) { client }
+          command.stub(:do_table_import)
+        end
+
+        context 'set --auto-create-table' do
+          let(:option) {
+            List::CommandParser.new('table:import', [], %w(db_name table_name path), false, [db_name, table_name, 'path', '--auto-create-table'], true)
+          }
+
+          it 'create table' do
+            command.should_receive(:create_database_and_table_if_not_exist).with(client, db_name, table_name)
+
+            command.table_import(option)
+          end
+        end
+
+        context 'not set --auto-create-table' do
+          let(:option) {
+            List::CommandParser.new('table:import', [], %w(db_name table_name path), false, [db_name, table_name, 'path'], true)
+          }
+
+          it 'not create table' do
+            command.should_not_receive(:create_database_and_table_if_not_exist)
+
+            command.table_import(option)
+          end
+        end
+      end
+    end
   end
 end

data/spec/td/common_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'td/command/common'
+require 'td/client/api_error'
 
 module TreasureData::Command
   describe 'humanize_bytesize' do
@@ -189,4 +190,105 @@ module TreasureData::Command
       end
     }
   end
+
+  describe '#create_database_and_table_if_not_exist' do
+    let(:command_class) { Class.new { include TreasureData::Command } }
+    let(:command)       { command_class.new }
+    let(:client)        { double(:client) }
+    let(:database_name) { 'database' }
+    let(:table_name)    { 'table1' }
+
+    let(:stderr_io) { StringIO.new }
+
+    subject(:call_create_database_and_table_if_not_exist) {
+      stderr = $stderr.dup
+      begin
+        $stderr = stderr_io
+
+        command.__send__(:create_database_and_table_if_not_exist, client, database_name, table_name)
+      ensure
+        $stderr = stderr
+      end
+    }
+
+    describe 'create database' do
+      before do
+        client.stub(:create_log_table)
+      end
+
+      context 'client.database success' do
+        it 'not call client.create_database' do
+          client.should_receive(:database).with(database_name)
+          client.should_not_receive(:create_database)
+
+          call_create_database_and_table_if_not_exist
+          expect(stderr_io.string).not_to include "Database '#{database_name}'"
+        end
+      end
+
+      context 'client.database raise NotFoundError' do
+        before do
+          client.should_receive(:database).with(database_name).and_return { raise TreasureData::NotFoundError }
+        end
+
+        context 'craet_database success' do
+          it 'call client.create_database' do
+            client.should_receive(:create_database).with(database_name)
+
+            call_create_database_and_table_if_not_exist
+            expect(stderr_io.string).to include "Database '#{database_name}'"
+          end
+        end
+
+        context 'craet_database raise AlreadyExistsError' do
+          it 'resuce in method' do
+            client.should_receive(:create_database).with(database_name).and_return { raise TreasureData::AlreadyExistsError }
+
+            expect {
+              call_create_database_and_table_if_not_exist
+            }.not_to raise_error
+
+            expect(stderr_io.string).not_to include "Database '#{database_name}'"
+          end
+        end
+      end
+
+      context 'client.database raise ForbiddenError' do
+        it 'not call client.create_database' do
+          client.should_receive(:database).with(database_name).and_return { raise TreasureData::ForbiddenError }
+          client.should_not_receive(:create_database)
+
+          call_create_database_and_table_if_not_exist
+          expect(stderr_io.string).not_to include "Database '#{database_name}'"
+        end
+      end
+    end
+
+    describe 'create table' do
+      before do
+        client.stub(:database)
+      end
+
+      context 'create_log_table success' do
+        it 'show message' do
+          client.should_receive(:create_log_table).with(database_name, table_name)
+
+          call_create_database_and_table_if_not_exist
+          expect(stderr_io.string).to include "Table '#{database_name}.#{table_name}'"
+        end
+      end
+
+      context 'create_log_table raise AlreadyExistsError' do
+        it 'resuce in method' do
+          client.should_receive(:create_log_table).with(database_name, table_name).and_return { raise TreasureData::AlreadyExistsError }
+
+          expect {
+            call_create_database_and_table_if_not_exist
+          }.not_to raise_error
+
+          expect(stderr_io.string).not_to include "Table '#{database_name}.#{table_name}'"
+        end
+      end
+    end
+  end
 end

data/td.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.72"
+  gem.add_dependency "td-client", "~> 0.8.73"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 1.1.7"
   gem.add_dependency "zip-zip", "~> 0.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.11.12
+  version: 0.11.13
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-27 00:00:00.000000000 Z
+date: 2015-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -102,14 +102,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.72
+        version: 0.8.73
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.72
+        version: 0.8.73
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement