td 0.12.0 → 0.13.0

data/spec/td/command/import_spec.rb

@@ -178,5 +178,184 @@ module TreasureData::Command
         expect(stderr_io.string).to include import_name
       end
     end
+
+    describe '#import_config' do
+      include_context 'quiet_out'
+
+      let :command do
+        Class.new { include TreasureData::Command }.new
+      end
+      let :option do
+        List::CommandParser.new("import:config", args, [], nil, arguments, true)
+      end
+      let :out_file do
+        Tempfile.new("seed.yml").tap {|s| s.close }
+      end
+      let(:endpoint) { 'http://example.com' }
+      let(:apikey) { '1234ABCDEFGHIJKLMN' }
+
+      before do
+        TreasureData::Config.stub(:endpoint).and_return(endpoint)
+        TreasureData::Config.stub(:apikey).and_return(apikey)
+      end
+
+      context 'unknown format' do
+        let(:args) { ['url'] }
+        let(:arguments) { ['localhost', '--format', 'msgpack', '-o', out_file.path] }
+
+        it 'exit command' do
+          expect {
+            command.import_config(option)
+          }.to raise_error TreasureData::Command::ParameterConfigurationError
+        end
+      end
+
+      context 'support format' do
+        let(:td_output_config) {
+          {
+            'type' => 'td',
+            'endpoint' => TreasureData::Config.endpoint,
+            'apikey' =>  TreasureData::Config.apikey,
+            'database' => '',
+            'table' => '',
+          }
+        }
+
+        before do
+          command.import_config(option)
+        end
+
+        subject(:generated_config) { YAML.load_file(out_file.path) }
+
+        %w(csv tsv).each do |format|
+          context "--format #{format}" do
+            let(:args) { ['url'] }
+            context 'use path' do
+              let(:path_prefix)   { 'path/to/prefix_' }
+              let(:arguments)     { ["#{path_prefix}*.#{format}", '--format', format, '-o', out_file.path] }
+
+              it 'generate configuration file' do
+                expect(generated_config).to eq({
+                  'in' => {
+                    'type'        => 'file',
+                    'decorders'   => [{'type' => 'gzip'}],
+                    'path_prefix' => path_prefix,
+                  },
+                  'out' => td_output_config
+                })
+              end
+            end
+
+            context 'use s3 scheme' do
+              let(:s3_access_key) { 'ABCDEFGHIJKLMN' }
+              let(:s3_secret_key) { '1234ABCDEFGHIJKLMN' }
+              let(:buckt_name)    { 'my_bucket' }
+              let(:path_prefix)   { 'path/to/prefix_' }
+              let(:s3_url)        { "s3://#{s3_access_key}:#{s3_secret_key}@/#{buckt_name}/#{path_prefix}*.#{format}" }
+              let(:arguments)     { [s3_url, '--format', format, '-o', out_file.path] }
+
+              it 'generate configuration file' do
+                expect(generated_config).to eq({
+                  'in' => {
+                    'type'              => 's3',
+                    'access_key_id'     => s3_access_key,
+                    'secret_access_key' => s3_secret_key,
+                    'bucket'            => buckt_name,
+                    'path_prefix'       => path_prefix,
+                  },
+                  'out' => {
+                    'mode' => 'append'
+                  }
+                })
+              end
+            end
+          end
+        end
+
+        context 'format is mysql' do
+          let(:format)   { 'mysql' }
+          let(:host)     { 'localhost' }
+          let(:database) { 'database' }
+          let(:user)     { 'my_user' }
+          let(:password) { 'my_password' }
+          let(:table)    { 'my_table' }
+
+          let(:expected_config) {
+            {
+              'in' => {
+                'type'     => 'mysql',
+                'host'     => host,
+                'port'     => port,
+                'database' => database,
+                'user'     => user,
+                'password' => password,
+                'table'    => table,
+                'select'   => "*",
+              },
+              'out' => td_output_config
+            }
+          }
+
+          context 'like import:prepare arguments' do
+            let(:args) { ['url'] }
+            let(:arguments) { [table, '--db-url', mysql_url, '--db-user', user, '--db-password', password, '--format', 'mysql', '-o', out_file.path] }
+
+            context 'scheme is jdbc' do
+              let(:port)      { 3333 }
+              let(:mysql_url) { "jdbc:mysql://#{host}:#{port}/#{database}" }
+
+              it 'generate configuration file' do
+                expect(generated_config).to eq expected_config
+              end
+            end
+
+            context 'scheme is mysql' do
+              context 'with port' do
+                let(:port)      { 3333 }
+                let(:mysql_url) { "mysql://#{host}:#{port}/#{database}" }
+
+                it 'generate configuration file' do
+                  expect(generated_config).to eq expected_config
+                end
+              end
+
+              context 'without port' do
+                let(:mysql_url) { "mysql://#{host}/#{database}" }
+                let(:port) { 3306 }
+
+                it 'generate configuration file' do
+                  expect(generated_config).to eq expected_config
+                end
+              end
+            end
+          end
+
+          context 'like import:upload arguments' do
+            let(:args)      { ['session', 'url'] }
+            let(:arguments) { ['session', table, '--db-url', mysql_url, '--db-user', user, '--db-password', password, '--format', 'mysql', '-o', out_file.path] }
+            let(:mysql_url) { "jdbc:mysql://#{host}/#{database}" }
+            let(:port)      { 3306 }
+
+            it 'generate configuration file' do
+              expect(generated_config).to eq expected_config
+            end
+          end
+        end
+      end
+
+      context 'not migrate options' do
+        %w(--columns --column-header).each do |opt|
+          context "with #{opt}" do
+            let(:args) { ['url'] }
+            let(:arguments)     { ["path/to/prefix_*.csv", '--format', 'csv', opt, 'col1,col2'] }
+
+            it "#{opt} is not migrate" do
+              expect { command.import_config(option) }.not_to raise_error
+              expect(stderr_io.string).to include 'not migrate. Please, edit config file after execute guess commands.'
+            end
+          end
+        end
+      end
+    end
   end
 end

data/spec/td/command/job_spec.rb

@@ -146,7 +146,7 @@ module TreasureData::Command
 
           it 'supports json output' do
             command.send(:show_result, job, file, nil, 'json', { header: true })
-            File.read(file.path).should == %Q([[null,2.0,{"key":3}]])
+            File.read(file.path).should == %Q([{"c0":null,"c1":2.0,"v":{"key":3},"c3":null}])
           end
 
           it 'supports csv output' do

data/spec/td/command/table_spec.rb

@@ -217,5 +217,118 @@ module TreasureData::Command
         end
       end
     end
+
+    describe '#table_rename' do
+      include_context 'quiet_out'
+
+      let(:db_name)         { 'database' }
+      let(:from_table_name) { 'from_table' }
+      let(:dest_table_name) { 'dest_table' }
+      let(:client)          { double('client') }
+      let(:database)        { double('database') }
+      let(:command)         { Class.new { include TreasureData::Command }.new }
+      let(:option) {
+        List::CommandParser.new('table:rename', [], %w(db_name from_table_name dest_table_name), false, cmd_args, true)
+      }
+
+      before do
+        command.stub(:get_client) { client }
+        command.stub(:get_database).with(client, db_name).and_return(database)
+      end
+
+      context "from table isn't exists" do
+        let(:cmd_args) { [db_name, from_table_name, dest_table_name] }
+
+        before do
+          database.stub(:table).with(from_table_name).and_return { raise }
+        end
+
+        it "can't rename table" do
+          command.should_receive(:exit).with(1).and_return { raise CallSystemExitError }
+
+          expect { command.table_rename(option) }.to raise_error
+          expect(stderr_io.string).to     include(from_table_name)
+          expect(stderr_io.string).not_to include(dest_table_name)
+        end
+      end
+
+      context 'overwrite is false' do
+        let(:cmd_args) { [db_name, from_table_name, dest_table_name] }
+
+        before do
+          database.stub(:table).with(from_table_name).and_return
+        end
+
+        context "dest_table isn't exists" do
+          before do
+            database.stub(:table).with(dest_table_name).and_return { raise }
+          end
+
+          it 'create dest table and rename table' do
+            client.should_receive(:create_log_table).with(db_name, dest_table_name)
+            client.should_receive(:swap_table).with(db_name, from_table_name, dest_table_name)
+            client.should_receive(:delete_table).with(db_name, from_table_name)
+
+            command.table_rename(option)
+
+            expect(stderr_io.string).to include(from_table_name)
+            expect(stderr_io.string).to include(dest_table_name)
+          end
+        end
+
+        context "dest_tableis is exist" do
+          before do
+            database.stub(:table).with(dest_table_name).and_return
+          end
+
+          it "can't rename table" do
+            command.should_receive(:exit).with(1).and_return { raise CallSystemExitError }
+
+            expect { command.table_rename(option) }.to raise_error
+            expect(stderr_io.string).not_to include(from_table_name)
+            expect(stderr_io.string).to     include(dest_table_name)
+          end
+        end
+      end
+
+      context 'overwrite is true' do
+        let(:cmd_args) { [db_name, from_table_name, dest_table_name, '--overwrite'] }
+
+        before do
+          database.stub(:table).with(from_table_name).and_return
+        end
+
+        context "dest_table isn't exists" do
+          before do
+            database.stub(:table).with(dest_table_name).and_return { raise }
+          end
+
+          it 'create dest table and rename table' do
+            client.should_receive(:create_log_table).with(db_name, dest_table_name)
+            client.should_receive(:swap_table).with(db_name, from_table_name, dest_table_name)
+            client.should_receive(:delete_table).with(db_name, from_table_name)
+
+            command.table_rename(option)
+            expect(stderr_io.string).to include(from_table_name)
+            expect(stderr_io.string).to include(dest_table_name)
+          end
+        end
+
+        context "dest_tableis is exist" do
+          before do
+            database.stub(:table).with(dest_table_name).and_return
+          end
+
+          it 'overwrite dest table' do
+            client.should_receive(:swap_table).with(db_name, from_table_name, dest_table_name)
+            client.should_receive(:delete_table).with(db_name, from_table_name)
+
+            command.table_rename(option)
+            expect(stderr_io.string).to include(from_table_name)
+            expect(stderr_io.string).to include(dest_table_name)
+          end
+        end
+      end
+    end
   end
 end

data/spec/td/common_spec.rb

@@ -144,7 +144,29 @@ module TreasureData::Command
       size_increments = 2
       curr_size = 0
       while (curr_size += size_increments) < size do
-        indicator.update(size_increments)
+        indicator.update(curr_size)
+        sleep(0.05)
+      end
+      indicator.finish
+    end
+
+    it "shows in only current byte size if total is nil" do
+      indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", nil)
+      size_increments = 2
+      curr_size = 0
+      while (curr_size += size_increments) < 200 do
+        indicator.update(curr_size)
+        sleep(0.05)
+      end
+      indicator.finish
+    end
+
+    it "shows in only current byte size if total is 0" do
+      indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", 0)
+      size_increments = 2
+      curr_size = 0
+      while (curr_size += size_increments) < 200 do
+        indicator.update(curr_size)
         sleep(0.05)
       end
       indicator.finish

data/spec/td/compact_format_yamler_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+require 'td/compact_format_yamler'
+
+module TreasureData
+  describe TreasureData::CompactFormatYamler do
+    describe '.dump' do
+      let(:data) {
+        {
+          'a' => {
+            'b' => {
+              'c' => 1,
+              'd' => 'e'
+            },
+            'f' => [1, 2, 3]
+          }
+        }
+      }
+
+      let(:comapct_format_yaml) {
+        <<-EOS
+---
+a:
+  b: {c: 1, d: e}
+  f:
+  - 1
+  - 2
+  - 3
+        EOS
+      }
+
+      subject { TreasureData::CompactFormatYamler.dump data }
+
+      it 'use compact format for deepest Hash' do
+        expect(subject).to eq comapct_format_yaml
+      end
+    end
+  end
+end

data/spec/td/connector_config_normalizer_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+require 'td/connector_config_normalizer'
+require 'td/client/api_error'
+
+module TreasureData
+  describe ConnectorConfigNormalizer do
+    describe '#normalized_config' do
+      subject { TreasureData::ConnectorConfigNormalizer.new(config).normalized_config }
+
+      context 'has key :in' do
+        context 'without :out, :exec' do
+          let(:config) { {'in' => {'type' => 's3'}} }
+
+          it { expect(subject).to eq config.merge('out' => {}, 'exec' => {}, 'filters' => []) }
+        end
+
+        context 'with :out, :exec, :filters' do
+          let(:config) {
+            {
+              'in'      => {'type' => 's3'},
+              'out'     => {'mode' => 'append'},
+              'exec'    => {'guess_plugins' => ['json', 'query_string']},
+              'filters' => [{'type' => 'speedometer'}]
+            }
+          }
+
+          it { expect(subject).to eq config }
+        end
+      end
+
+      context 'has key :config' do
+        context 'with :in' do
+          let(:config) {
+            { 'config' =>
+              {
+                'in'   => {'type' => 's3'},
+                'out'  => {'mode' => 'append'},
+                'exec' => {'guess_plugins' => ['json', 'query_string']},
+                'filters' => [{'type' => 'speedometer'}]
+              }
+            }
+          }
+
+          it { expect(subject).to eq config['config'] }
+
+        end
+
+        context 'without :in' do
+          let(:config) { {'config' => {'type' => 's3'}} }
+
+          it { expect(subject).to eq({'in' => config['config'], 'out' => {}, 'exec' => {}, 'filters' => []}) }
+        end
+      end
+
+      context 'does not have key :in or :config' do
+        let(:config) { {'type' => 's3'} }
+
+        it { expect(subject).to eq({'in' => config, 'out' => {}, 'exec' => {}, 'filters' => []}) }
+      end
+    end
+  end
+end

data/td.gemspec

@@ -21,10 +21,11 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.75"
+  gem.add_dependency "td-client", "~> 0.8.76"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 1.1.7"
   gem.add_dependency "zip-zip", "~> 0.3"
+  gem.add_dependency "ruby-progressbar", "~> 1.7.5"
   gem.add_development_dependency "rake", "~> 0.9"
   gem.add_development_dependency "rspec", "~> 2.11.0"
   gem.add_development_dependency "simplecov", "~> 0.10.0"