td 0.11.9 → 0.11.10

data/spec/td/command/job_spec.rb

@@ -22,9 +22,81 @@ module TreasureData::Command
           @result_size = 3
           @status = 'success'
         end
+        job.stub(:result_format) # for msgpack, msgpack.gz
         job
       end
 
+      describe "using tempfile" do
+        let(:tempfile) { "#{file.path}.tmp" }
+
+        subject { command.send(:show_result, job, file, nil, format) }
+
+        context "format: json" do
+          let(:format) { "json" }
+
+          it do
+            FileUtils.should_receive(:mv).with(tempfile, file.path)
+            subject
+          end
+          it do
+            command.should_receive(:open_file).with(tempfile, "w")
+            subject
+          end
+        end
+
+        context "format: csv" do
+          let(:format) { "csv" }
+
+          it do
+            FileUtils.should_receive(:mv).with(tempfile, file.path)
+            subject
+          end
+          it do
+            command.should_receive(:open_file).with(tempfile, "w")
+            subject
+          end
+        end
+
+        context "format: tsv" do
+          let(:format) { "tsv" }
+
+          it do
+            FileUtils.should_receive(:mv).with(tempfile, file.path)
+            subject
+          end
+          it do
+            command.should_receive(:open_file).with(tempfile, "w")
+            subject
+          end
+        end
+
+        context "format: msgpack" do
+          let(:format) { "msgpack" }
+
+          it do
+            FileUtils.should_receive(:mv).with(tempfile, file.path)
+            subject
+          end
+          it do
+            command.should_receive(:open_file).with(tempfile, "wb")
+            subject
+          end
+        end
+
+        context "format: msgpack.gz" do
+          let(:format) { "msgpack.gz" }
+
+          it do
+            FileUtils.should_receive(:mv).with(tempfile, file.path)
+            subject
+          end
+          it do
+            command.should_receive(:open_file).with(tempfile, "wb")
+            subject
+          end
+        end
+      end
+
       context 'result without nil' do
         it 'supports json output' do
           command.send(:show_result, job, file, nil, 'json')
@@ -238,6 +310,81 @@ module TreasureData::Command
       end
     end
 
+    describe '#job_list' do
+      subject do
+        backup = $stdout.dup
+        buf = StringIO.new
+        op = List::CommandParser.new("job:list", [], [:max], nil, [], true)
+
+        begin
+          $stdout = buf
+          command.job_list(op)
+          buf.string
+        ensure
+          $stdout = backup
+        end
+      end
+
+      let(:job_id) { "12345" }
+
+      let :job_class do
+        Struct.new(:job_id,
+                   :status,
+                   :type,
+                   :db_name,
+                   :priority,
+                   :retry_limit,
+                   :result_url,
+                   :query,
+                   :start_at,
+                   :end_at,
+                   :cpu_time,
+                   :result_size,
+                   :duration
+                  )
+      end
+
+      let :start_at do
+        Time.now
+      end
+
+      let :jobs do
+        [job_class.new(job_id,
+                       nil,
+                       :hive,
+                       "db_name",
+                       1,
+                       1,
+                       "test_url",
+                       "test_qury",
+                       start_at,
+                       start_at + 10,
+                       1,
+                       3,
+                       100
+                      )] * 3
+      end
+
+      before do
+        client = Object.new
+        client.stub(:jobs).and_return(jobs)
+        command.stub(:get_client).and_return(client)
+      end
+
+      it 'should display all job list' do
+        expect(subject).to eq <<ACTUAL
++-------+--------+---------------------------+-------------+-------------------+------------+----------+----------+------+----------+---------------+----------+
+| JobID | Status | Start                     | Elapsed     | CPUTime           | ResultSize | Priority | Result   | Type | Database | Query         | Duration |
++-------+--------+---------------------------+-------------+-------------------+------------+----------+----------+------+----------+---------------+----------+
+| 12345 |        | #{start_at} |         10s |             001ms | 3 B        | HIGH     | test_url | hive | db_name  | test_qury ... | 00:01:40 |
+| 12345 |        | #{start_at} |         10s |             001ms | 3 B        | HIGH     | test_url | hive | db_name  | test_qury ... | 00:01:40 |
+| 12345 |        | #{start_at} |         10s |             001ms | 3 B        | HIGH     | test_url | hive | db_name  | test_qury ... | 00:01:40 |
++-------+--------+---------------------------+-------------+-------------------+------------+----------+----------+------+----------+---------------+----------+
+3 rows in set
+ACTUAL
+      end
+    end
+
     describe 'multibyte chars' do
       let :multibyte_string do
         # Originally a Windows-31J but in UTF-8 like msgpack-ruby populates

data/spec/td/command/sched_spec.rb

@@ -7,22 +7,144 @@ require 'td/client/model'
 require 'time'
 
 module TreasureData::Command
+  describe TreasureData::Command do
+    let(:client) { Object.new }
 
-  describe 'sched_history' do
-    it 'runs' do
-      client = Object.new
-      time = Time.now.xmlschema
-      job_params = ['job_id', :type, 'query', 'status', nil, nil, time, time, 123, 456]
-      job1 = TreasureData::ScheduledJob.new(client, '2015-02-17 13:22:52 +0900', *job_params)
-      job2 = TreasureData::ScheduledJob.new(client, nil, *job_params)
-      client.stub(:schedules).and_return([])
-      client.stub(:history).and_return([job1, job2])
-      command = Class.new { include TreasureData::Command }.new
+    let :job_params do
+      ['job_id', :type, 'query', 'status', nil, nil, time, time, 123, 456]
+    end
+
+    let(:job1) { TreasureData::ScheduledJob.new(client, '2015-02-17 13:22:52 +0900', *job_params) }
+    let(:job2) { TreasureData::ScheduledJob.new(client, nil, *job_params) }
+    let(:time) { Time.now.xmlschema }
+    let(:command) { Class.new { include TreasureData::Command }.new }
+    let(:argv) { [] }
+    let(:schedules) { [] }
+    let(:op) { List::CommandParser.new('sched:last_job', %w[], %w[], false, argv, []) }
+
+    before do
+      client.stub(:schedules).and_return(schedules)
       command.stub(:get_client).and_return(client)
-      op = List::CommandParser.new('sched:history', %w[], %w[], false, [], [])
-      expect {
-        command.sched_history(op)
-      }.to_not raise_exception
+    end
+
+    describe 'sched_history' do
+      before do
+        client.stub(:history).and_return(history)
+      end
+
+      let(:history) { [job1, job2] }
+
+      it 'runs' do
+        expect {
+          command.sched_history(op)
+        }.to_not raise_exception
+      end
+    end
+
+    describe 'sched_result' do
+      subject { command.sched_result(op) }
+
+      before do
+        command.stub(:get_history).with(client, nil, (back_number - 1), back_number).and_return(history)
+      end
+
+      shared_examples_for("passing argv and job_id to job:show") do
+        it "invoke 'job:show [original argv] [job id]'" do
+          TreasureData::Command::Runner.any_instance.should_receive(:run).with(["job:show", *show_arg, job_id])
+          subject
+        end
+      end
+
+      context "history exists" do
+
+        let(:job_id) { history.first.job_id }
+
+        context 'without --last option' do
+          let(:history) { [job1] }
+          let(:back_number) { 1 }
+
+          let(:argv) { %w(--last --format csv) }
+          let(:show_arg) { %w(--format csv) }
+          it_behaves_like "passing argv and job_id to job:show"
+        end
+
+        context '--last witout Num' do
+          let(:history) { [job1] }
+          let(:back_number) { 1 }
+
+          let(:argv) { %w(--last --format csv) }
+          let(:show_arg) { %w(--format csv) }
+          it_behaves_like "passing argv and job_id to job:show"
+        end
+
+        context '--last 1' do
+          let(:history) { [job1] }
+          let(:back_number) { 1 }
+
+          let(:argv) { %w(--last 1 --format csv) }
+          let(:show_arg) { %w(--format csv) }
+          it_behaves_like "passing argv and job_id to job:show"
+        end
+
+        context '--last 1 and --limit 1' do
+          let(:history) { [job1] }
+          let(:back_number) { 1 }
+
+          let(:argv) { %w(--last 1 --format csv --limit 1) }
+          let(:show_arg) { %w(--format csv --limit 1) }
+          it_behaves_like "passing argv and job_id to job:show"
+        end
+
+        context '--last 2 after format option' do
+          let(:history) { [job2] }
+          let(:back_number) { 2 }
+
+          let(:argv) { %w(--format csv --last 2 ) }
+          let(:show_arg) { %w(--format csv) }
+          it_behaves_like "passing argv and job_id to job:show"
+        end
+
+        context '--last 3(too back over)' do
+          let(:history) { [] }
+          let(:back_number) { 3 }
+
+          let(:argv) { %w(--last 3 --format csv) }
+          it 'raise ' do
+            expect {
+              command.sched_result(op)
+            }.to raise_exception, "No jobs available for this query. Refer to 'sched:history'"
+          end
+        end
+
+        context '--last WRONGARG(not a number)' do
+          let(:history) { [job1] }
+          let(:back_number) { 1 }
+
+          let(:argv) { %w(--last TEST --format csv) }
+
+          it "exit with 1" do
+            begin
+              command.sched_result(op)
+            rescue SystemExit => e
+              expect(e.status).to eq 1
+            end
+          end
+        end
+      end
+
+      context "history dose not exists" do
+        let(:back_number) { 1 }
+        let(:history) { [] }
+        before { client.stub(:history) { raise TreasureData::NotFoundError } }
+
+        it "exit with 1" do
+          begin
+            subject
+          rescue SystemExit => e
+            expect(e.status).to eq 1
+          end
+        end
+      end
     end
   end
 end

data/spec/td/command/table_spec.rb

@@ -130,11 +130,8 @@ module TreasureData::Command
         before do
           create_tables = lambda {|db_name|
             (1..6).map {|i|
-              # NOTE: TreasureData::Helpers.format_with_delimiter uses `gsub!` to their argument
-              #       the argument (in our case, `number_raw`) will be rewritten by them
-              #       To avoid that behavior, pass `number_raw.dup` instead of `number_raw`
               schema = TreasureData::Schema.new.from_json(JSON.parse('[]'))
-              TreasureData::Table.new(client, db_name, db_name + "_table_#{i}", 'log', schema, number_raw.dup, Time.now.to_i, Time.now.to_i, 0, nil, nil, nil, nil, nil)
+              TreasureData::Table.new(client, db_name, db_name + "_table_#{i}", 'log', schema, number_raw, Time.now.to_i, Time.now.to_i, 0, nil, nil, nil, nil, nil)
             }
           }
           db_tables = create_tables.call(db.name)

data/spec/td/fixture/bulk_load.yml

@@ -0,0 +1,24 @@
+---
+config:
+  type: s3
+  access_key_id: ACCESS_KEY
+  secret_access_key: SECRET_ACCESS
+  bucket: test-bucket
+  path_prefix: dummy.csv.gz
+  parser:
+    charset: ISO-8859-9
+    newline: CRLF
+    type: csv
+    delimiter: ","
+    quote: ''
+    escape: ''
+    skip_header_lines: 1
+    columns:
+    - name: foo
+      type: long
+    - name: bar
+      type: long
+    - name: baz
+      type: long
+  decoders:
+  - type: gzip

data/td.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.69"
+  gem.add_dependency "td-client", "~> 0.8.70"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"

metadata

@@ -1,183 +1,204 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.11.9
+  version: 0.11.10
+  prerelease: 
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-16 00:00:00.000000000 Z
+date: 2015-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
-    - - "<"
+    - - <
       - !ruby/object:Gem::Version
         version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - "!="
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
-    - - "<"
+    - - <
       - !ruby/object:Gem::Version
         version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.69
+        version: 0.8.70
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.69
+        version: 0.8.70
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.21
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.9
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.4
 description: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
@@ -187,7 +208,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - ChangeLog
 - Gemfile
 - README.rdoc
@@ -211,6 +232,7 @@ files:
 - lib/td/command/import.rb
 - lib/td/command/job.rb
 - lib/td/command/list.rb
+- lib/td/command/options.rb
 - lib/td/command/password.rb
 - lib/td/command/query.rb
 - lib/td/command/result.rb
@@ -238,11 +260,13 @@ files:
 - spec/file_reader/shared_context.rb
 - spec/file_reader_spec.rb
 - spec/spec_helper.rb
+- spec/td/command/connector_spec.rb
 - spec/td/command/import_spec.rb
 - spec/td/command/job_spec.rb
 - spec/td/command/sched_spec.rb
 - spec/td/command/table_spec.rb
 - spec/td/common_spec.rb
+- spec/td/fixture/bulk_load.yml
 - spec/td/fixture/ca.cert
 - spec/td/fixture/server.cert
 - spec/td/fixture/server.key
@@ -253,26 +277,30 @@ files:
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -1990136432195876879
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.23.2
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb
@@ -282,11 +310,13 @@ test_files:
 - spec/file_reader/shared_context.rb
 - spec/file_reader_spec.rb
 - spec/spec_helper.rb
+- spec/td/command/connector_spec.rb
 - spec/td/command/import_spec.rb
 - spec/td/command/job_spec.rb
 - spec/td/command/sched_spec.rb
 - spec/td/command/table_spec.rb
 - spec/td/common_spec.rb
+- spec/td/fixture/bulk_load.yml
 - spec/td/fixture/ca.cert
 - spec/td/fixture/server.cert
 - spec/td/fixture/server.key

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 35121e210f89e8af8d88f3de09c5da1a46f465f5
-  data.tar.gz: 2ed8a007214c1861d9653ad88a6e204b11959487
-SHA512:
-  metadata.gz: e29254a97c4170455119d3d0d11a8c026d05b431f3fe58ad69bbaeb7f6837805fcdbe51fdd91c8ebe3ae6a9c7077f3312cd681a0f865a3bf2eddb43399446be0
-  data.tar.gz: 908f511b6ed2f6b293cdc809b203a367869607f7fd0d130c5f85919cc5d0462fec372a214c31df2ecde30cde248804c173fb9a5d32136d84f605d5557fc09cb3