td 0.10.83 → 0.10.84

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c7c60a874b7228266a8436132aae4d0e2438b3e5
+  data.tar.gz: 9caacd3a0d3065d8fd4dca7bc7bae1cd29b9bbd5
+SHA512:
+  metadata.gz: baa729bcbfb73760f2c93cc70e6fefccc69b1cae9a41c62e7c869372bf4124975649e0abffb5db15b79156bfb59e59ab52875634bf179da86fab129979c7cdab
+  data.tar.gz: 10328c283f1b8a9a3e4d2b9ad23187da08b0b0391ae237fdd5fbe8ad4c610784f0d8a2f2ca8d00c758f5afac757426965545e5811b406df943769a826b9a27d7

data/ChangeLog

@@ -1,3 +1,11 @@
+== 2013-07-25 version 0.10.84
+
+* bulk_import now checks first argument is correct session name
+* Convert bignum into string type. Please use cast in your query.
+* Improve error message of export -F option
+* td-client v0.8.53
+
+
 == 2013-07-03 version 0.10.83
 
 * Add -x option to query, exlude result from automatically being downloaded

data/lib/td/command/bulk_import.rb

@@ -129,6 +129,13 @@ module Command
 
     name, *files = op.cmd_parse
 
+    # validate the session
+    bi = get_client.bulk_imports.find {|bi| name == bi.name }
+    unless bi
+      $stderr.puts "Bulk import session '#{name}' does not exist. Please check the first argument of the command."
+      exit 1
+    end
+
     parallel = 1 if parallel <= 1
     parallel = 8 if parallel >= 8
 
@@ -320,7 +327,6 @@ module Command
     $stderr.puts "Bulk import session '#{name}' is unfrozen."
   end
 
-
   PART_SPLIT_SIZE = 16*1024*1024
 
   def bulk_import_prepare_parts(op)
@@ -364,6 +370,7 @@ module Command
         $stderr.puts "#{reason}"
       end
     }
+    has_bignum = false
 
     # TODO multi process
     files.each {|ifname|
@@ -387,7 +394,13 @@ module Command
               $stderr.puts "  sample: #{Time.at(t).utc} #{Yajl.dump(record)}"
             end
 
-            zout.write(record.to_msgpack)
+            entry = begin
+                      record.to_msgpack
+                    rescue RangeError
+                      has_bignum = true
+                      TreasureData::API.normalized_msgpack(record)
+                    end
+            zout.write(entry)
             record_num += 1
 
             if out.size > split_size
@@ -406,6 +419,8 @@ module Command
         $stderr.puts "  #{ifname}: #{record_num} entries."
       }
     }
+
+    $stderr.puts normalized_message if has_bignum
   end
 
   def bulk_import_upload_parts2(op)

data/lib/td/command/common.rb

@@ -46,6 +46,14 @@ module Command
     end
   end
 
+  def normalized_message
+    <<EOS
+Your event has large number larger than 2^64.
+These numbers are converted into string type.
+So you should use cast operator, e.g. cast(v['key'] as decimal), in your query.
+EOS
+  end
+
   #def cmd_render_tree(nodes, *opts)
   #  require 'hirb'
   #  Hirb::Helpers::Tree.render(nodes, *opts)

data/lib/td/command/export.rb

@@ -1,6 +1,7 @@
 
 module TreasureData
 module Command
+  SUPPORTED_FORMATS = %W[json.gz line-json.gz]
 
   def table_export(op)
     org = nil
@@ -9,7 +10,7 @@ module Command
     s3_bucket = nil
     aws_access_key_id = nil
     aws_secret_access_key = nil
-    file_format = "json.gz"
+    file_format = nil
 
     op.on('-g', '--org ORGANIZATION', "export the data under this organization") {|s|
       org = s
@@ -30,7 +31,8 @@ module Command
       aws_secret_access_key = s
     }
     op.on('-F', '--file-format FILE_FORMAT', 'file format for exported data, either json.gz (default) or line-json.gz') { |s| 
-      file_format = s if s == 'line-json.gz'
+      raise ArgumentError, "#{s} is not a supported file format" unless SUPPORTED_FORMATS.include?(s)
+      file_format = s
     }
 
     db_name, table_name = op.cmd_parse

data/lib/td/command/import.rb

@@ -136,8 +136,15 @@ module Command
 
     n = 0
     x = 0
+    has_bignum = false
     parser.call(file, path) {|record|
-      writer.write record.to_msgpack
+      entry = begin
+                record.to_msgpack
+              rescue RangeError
+                has_bignum = true
+                TreasureData::API.normalized_msgpack(record)
+              end
+      writer.write entry
 
       n += 1
       x += 1
@@ -176,7 +183,7 @@ module Command
     end
 
     puts "  imported #{n} entries from #{path}."
-
+    $stderr.puts normalized_message if has_bignum
   ensure
     out.close rescue nil
     writer.close rescue nil

data/lib/td/command/job.rb

@@ -130,13 +130,13 @@ module Command
 
     if wait && !job.finished?
       wait_job(job)
-      if job.success? && [:hive, :pig].include?(job.type) && !exclude
+      if job.success? && [:hive, :pig, :impala].include?(job.type) && !exclude
         puts "Result       :"
         show_result(job, output, format, render_opts)
       end
 
     else
-      if job.success? && [:hive, :pig].include?(job.type) && !exclude
+      if job.success? && [:hive, :pig, :impala].include?(job.type) && !exclude
         puts "Result       :"
         show_result(job, output, format, render_opts)
       end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.83'
+VERSION = '0.10.84'
 
 end

data/td.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1.0"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.52"
+  gem.add_dependency "td-client", "~> 0.8.53"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"

metadata

@@ -1,34 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.83
-  prerelease: 
+  version: 0.10.84
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-03 00:00:00.000000000 Z
+date: 2013-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -37,21 +35,20 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -60,7 +57,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -68,7 +64,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -76,23 +71,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -100,7 +92,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -108,23 +99,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.52
+        version: 0.8.53
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.52
+        version: 0.8.53
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -132,7 +120,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -140,7 +127,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -148,7 +134,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -156,7 +141,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -164,7 +148,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -172,7 +155,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -180,7 +162,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -188,7 +169,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -196,7 +176,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -280,33 +259,26 @@ files:
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2192841065955335729
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2192841065955335729
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb