td 0.10.96 → 0.10.97

data/ChangeLog

@@ -1,3 +1,10 @@
+== 2013-12-18 version 0.10.97
+
+* import:commit accepts --wait option.
+* import:jar_update command to download correct jar file
+* Fix no error message when impala query failed immediately
+
+
 == 2013-12-04 version 0.10.96
 
 * Add --format option to some subcommands to specify render format. json, csv and tsv.

data/build/update-td-import-java.sh

@@ -19,7 +19,7 @@ if [ -n "$chrev" ];then
 fi
 
 revname="$(git show --pretty=format:'%H %ad' | head -n 1)"
-vername="0.4.2-SNAPSHOT"
+vername="0.4.1-SNAPSHOT"
 
 mvn package -Dmaven.test.skip=true || exit 1
 echo "copy td-import-${vername}.jar"

data/java/VERSION

@@ -1 +1 @@
-0.4.2-SNAPSHOT
+0.4.1-SNAPSHOT

data/lib/td/command/bulk_import.rb

@@ -269,6 +269,12 @@ module Command
   end
 
   def bulk_import_commit(op)
+    wait = false
+
+    op.on('-w', '--wait', 'wait for finishing the commit', TrueClass) {|b|
+      wait = b
+    }
+
     name = op.cmd_parse
 
     client = get_client
@@ -276,6 +282,44 @@ module Command
     job = client.commit_bulk_import(name)
 
     $stderr.puts "Bulk import session '#{name}' started to commit."
+
+    if wait
+      wait_commit(name) # wait the commit
+    end
+  end
+
+  private
+  def wait_commit(name)
+    first_call = true
+    finished = false
+    max_error_counts = 10
+
+    while first_call || !finished
+      first_call = false
+      begin
+        sleep 5
+
+        # extract buik_import session
+        client = get_client
+        bis = client.bulk_imports
+        bi = bis.find {|bi| name == bi.name }
+        unless bi
+          $stderr.puts "Bulk import session '#{name}' does not exist."
+          exit 1
+        end
+
+        # check status of the bulk_import session
+        if bi.status == 'committed' || bi.status == 'ready'
+          finished = true
+        end
+      rescue Timeout::Error, SystemCallError, EOFError, SocketError
+        if max_error_counts <= 0
+          raise
+        end
+        max_error_counts -= 1
+        retry
+      end
+    end
   end
 
   def bulk_import_error_records(op)

data/lib/td/command/import.rb

@@ -50,7 +50,7 @@ module Command
       File.open(File.join(UPDATED_PATH, 'VERSION'), 'w') { |f| f.print "#{version} via import:jar_update" }
       File.open(File.join(UPDATED_PATH, 'td-import-java.version'), 'w') { |f| f.print "#{version} #{updated}" }
       File.open(File.join(UPDATED_PATH, 'td-import.jar'), 'wb') { |f|
-        f.print Updater.fetch("http://maven.treasure-data.com/com/treasure_data/td-import/#{version}/td-import-#{version}.jar")
+        f.print Updater.fetch("http://maven.treasure-data.com/com/treasure_data/td-import/#{version}/td-import-#{version}-jar-with-dependencies.jar")
       }
       puts "Installed td-import.jar #{version} into #{UPDATED_PATH}"
     else

data/lib/td/command/job.rb

@@ -196,14 +196,15 @@ module Command
   end
 
   private
-  def wait_job(job)
+  def wait_job(job, first_call = false)
     $stderr.puts "queued..."
 
     cmdout_lines = 0
     stderr_lines = 0
     max_error_counts = JOB_WAIT_MAX_RETRY_COUNT_ON_NETWORK_ERROR
 
-    until job.finished?
+    while first_call || !job.finished?
+      first_call = false
       begin
         sleep 2
         job.update_status!

data/lib/td/command/query.rb

@@ -111,8 +111,8 @@ module Command
     $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
     #$stderr.puts "See #{job.url} to see the progress."
 
-    if wait && !job.finished?
-      wait_job(job)
+    if wait
+      wait_job(job, true)
       puts "Status     : #{job.status}"
       if job.success? && !exclude
         puts "Result     :"

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.96'
+VERSION = '0.10.97'
 
 end

metadata

@@ -1,32 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.96
+  version: 0.10.97
+  prerelease: 
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-04 00:00:00.000000000 Z
+date: 2013-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -35,20 +37,21 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -57,6 +60,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -64,6 +68,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -71,20 +76,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -92,6 +100,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -99,6 +108,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -106,6 +116,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -113,6 +124,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -120,6 +132,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -127,6 +140,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -134,6 +148,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -141,6 +156,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -148,6 +164,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -155,6 +172,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -162,6 +180,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -169,6 +188,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -176,6 +196,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -258,26 +279,33 @@ files:
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -1410145252245092295
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -1410145252245092295
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 4be1e9e6df99daff7484f415d97c0e07b1f8323b
-  data.tar.gz: e5a02f18e5ddc92ce3853064982dac0276c78b51
-SHA512:
-  metadata.gz: 84a6b09a04a928bc8cd07230fc21e041d451bb43d42614821623507d3bfcd55ffaa4d93bdc8372178d11937a5830df5cabef97a82bd53cd6992032b036e9331e
-  data.tar.gz: 6aa3f40c432bc25e2a8430595f071e92388f58ffb47f4e472d113100e58dc440ed49ebd1b3d9886ba36950979456791cf5e248d918e6e7fefffe29d3d4c689e9