td 0.15.2 → 0.15.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d67f821df06a365e2fc2b35febb963900e683f0a
-  data.tar.gz: 5af9e2025dbb5a756e0e2f8b4d53b105613165cf
+  metadata.gz: 0664e06a537f0c5e0a8c191df507b03462a803dd
+  data.tar.gz: e03b262494754689483a2e132401efdc1f436b6d
 SHA512:
-  metadata.gz: 98afa7ab296af4840874998df5dc55a940083341d6bb83768b3353b33774ea8948e95fd21b4794a8c171a8e4f7fc92939a0328f44dd0eb546f8747f1ab00ab83
-  data.tar.gz: b6a0b37882e836f90f90cd62da7d7adfb438af1569e2eca7702df483a5ca843bf079338d020c0447372b4657eaa8fd922c2f8d94e2c5eb1253218712877a4816
+  metadata.gz: 65c89dc8dd648fb8ad188be1da9dd2a26e1b07b93ad1dfbc7119870e7d648f13290bb1828e0f087932afcd8103fe425ac5f2eff2fcbe7832a2ec083d56b4f3ba
+  data.tar.gz: 819bf1005d21dbb40e671f4e93dabfbff79ce0c948982e5734dfb75d640466197300a4857564455f802de3f6fe5f5df34d56d069b8c4704dcf7f47a06c79fea9

data/ChangeLog

@@ -1,3 +1,8 @@
+== 2017-06-08 version 0.15.3
+
+* add jsonl.gz format #153
+* add workflow:update command #178
+
 == 2016-11-29 version 0.15.2
 
 * Deprecate Pig query engine #172

data/contrib/completion/_td

@@ -16,7 +16,7 @@ subcommands=(
 
     "bulk_import\:list":"List bulk import sessions"
     "bulk_import\:show":"Show list of uploaded parts"
-    "bulk_import\:create":"Create a new bulk import session to the the table"
+    "bulk_import\:create":"Create a new bulk import session to the table"
     "bulk_import\:prepare_parts":"Convert files into part file format"
     #"bulk_import\:prepare_parts2":"Convert files into part file format"
     "bulk_import\:upload_part":"Upload or re-upload a file into a bulk import session"
@@ -84,6 +84,7 @@ subcommands=(
     "wf":"Run a workflow command"
     "workflow":"Run a workflow command"
     "workflow:reset":"Reset the workflow module"
+    "workflow:update":"Update the workflow module"
 
     # TODO: Add ACL related commands
 )

data/lib/td/command/bulk_import.rb

@@ -1,3 +1,4 @@
+require 'td/command/job'
 
 module TreasureData
 module Command

data/lib/td/command/export.rb

@@ -1,7 +1,8 @@
+require 'td/command/job'
 
 module TreasureData
 module Command
-  SUPPORTED_FORMATS = %W[json.gz line-json.gz tsv.gz]
+  SUPPORTED_FORMATS = %W[json.gz line-json.gz tsv.gz jsonl.gz]
   SUPPORTED_ENCRYPT_METHOD = %W[s3]
 
   def export_result(op)
@@ -22,7 +23,7 @@ module Command
     $stderr.puts "result export job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
 
-    if wait && !job.finished?
+    if wait
       wait_job(job)
       $stdout.puts "status     : #{job.status}"
     end
@@ -66,7 +67,11 @@ module Command
     op.on('-s', '--aws-secret-key SECRET_KEY', 'AWS secret access key to export data (required)') {|s|
       aws_secret_access_key = s
     }
-    op.on('-F', '--file-format FILE_FORMAT', 'file format for exported data, either json.gz (default) or line-json.gz, or tsv.gz') { |s|
+    op.on('-F', '--file-format FILE_FORMAT',
+          'file format for exported data.',
+          'Available formats are tsv.gz (tab-separated values per line) and jsonl.gz (JSON record per line).',
+          'The json.gz and line-json.gz formats are default and still available but only for backward compatibility purpose;',
+          '  use is discouraged because they have far lower performance.') { |s|
       raise ArgumentError, "#{s} is not a supported file format" unless SUPPORTED_FORMATS.include?(s)
       file_format = s
     }
@@ -121,7 +126,7 @@ module Command
     $stderr.puts "Export job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
 
-    if wait && !job.finished?
+    if wait
       wait_job(job)
       $stdout.puts "Status     : #{job.status}"
     end

data/lib/td/command/list.rb

@@ -246,7 +246,7 @@ module List
 
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', ['bulk_import:list']
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', ['bulk_import:show']
-  add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', ['bulk_import:create logs_201201 example_db event_logs']
+  add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the table', ['bulk_import:create logs_201201 example_db event_logs']
   add_list 'bulk_import:prepare_parts', %w[files_], 'Convert files into part file format', ['bulk_import:prepare_parts logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/']
   add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', ['bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz']
   add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', ['bulk_import:upload_parts parts/* --parallel 4']
@@ -261,7 +261,7 @@ module List
 
   add_list 'import:list', %w[], 'List bulk import sessions', ['import:list']
   add_list 'import:show', %w[name], 'Show list of uploaded parts', ['import:show']
-  add_list 'import:create', %w[name db table], 'Create a new bulk import session to the the table', ['import:create logs_201201 example_db event_logs']
+  add_list 'import:create', %w[name db table], 'Create a new bulk import session to the table', ['import:create logs_201201 example_db event_logs']
   add_list 'import:jar_version', %w[], 'Show import jar version', ['import:jar_version'], false
   add_list 'import:jar_update', %w[], 'Update import jar to the latest version', ['import:jar_update']
   add_list 'import:prepare', %w[files_], 'Convert files into part file format', ['import:prepare logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'], false
@@ -362,6 +362,7 @@ module List
 
   add_list 'workflow', %w[], 'Run a workflow command'
   add_list 'workflow:reset', %w[], 'Reset the workflow module'
+  add_list 'workflow:update', %w[version?], 'Update the workflow module'
   add_list 'workflow:version', %w[], 'Show workflow module version'
 
   # aliases

data/lib/td/command/query.rb

@@ -1,3 +1,4 @@
+require 'td/command/job'
 require 'td/command/options'
 
 module TreasureData

data/lib/td/command/table.rb

@@ -1,4 +1,5 @@
 require 'td/helpers'
+require 'td/command/job'
 
 module TreasureData
 module Command
@@ -276,9 +277,7 @@ module Command
     wait = false
     pool_name = nil
 
-    op.on('-t', '--to TIME', 'end time of logs to delete in Unix time multiple of 3600 (1 hour)',
-                             '  or Ruby time string format (e.g. \'2014-07-01 14:00:00 JST\') where',
-                             '  the minutes and seconds are required to be 0.') {|s|
+    op.on('-t', '--to TIME', 'end time of logs to delete in Unix time >0 and multiple of 3600 (1 hour)') {|s|
       if s.to_i.to_s == s
         # UNIX time
         to = s.to_i
@@ -287,9 +286,7 @@ module Command
         to = Time.parse(s).to_i
       end
     }
-    op.on('-f', '--from TIME', 'start time of logs to delete in Unix time multiple of 3600 (1 hour)',
-                               '  or Ruby time string format (e.g. \'2014-07-01 13:00:00 JST\') where',
-                               '  the minutes and seconds are required to be 0.') {|s|
+    op.on('-f', '--from TIME', 'start time of logs to delete in Unix time >0 and multiple of 3600 (1 hour)') {|s|
       if s.to_i.to_s == s
         from = s.to_i
       else
@@ -333,7 +330,7 @@ module Command
     $stderr.puts "Partial delete job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
 
-    if wait && !job.finished?
+    if wait
       wait_job(job)
       $stdout.puts "Status     : #{job.status}"
     end

data/lib/td/command/workflow.rb

@@ -66,8 +66,18 @@ module TreasureData
       }
     end
 
+    def workflow_update(op)
+      version = op.cmd_parse
+      $stdout << "Downloading workflow module #{version}..."
+      download_digdag(version)
+      $stdout.puts ' Done.'
+      return 0
+    end
+
     # "Factory reset"
     def workflow_reset(op)
+      op.cmd_parse # to show help
+
       $stdout << 'Removing workflow module...'
       FileUtils.rm_rf digdag_dir
       $stdout.puts ' Done.'
@@ -75,6 +85,8 @@ module TreasureData
     end
 
     def workflow_version(op)
+      op.cmd_parse # to show help
+
       unless File.exists?(digdag_cli_path)
         $stderr.puts('Workflow module not yet installed.')
         return 1
@@ -118,12 +130,17 @@ module TreasureData
       ENV.fetch('TD_WF_JAVA', '').strip
     end
 
-    def digdag_base_url
-      'http://toolbelt.treasure-data.com/digdag'
+    def digdag_base_url(version=nil)
+      base = 'http://toolbelt.treasure-data.com/digdag'
+      if version.to_s == ''
+        base
+      else
+        "#{base}-#{version}"
+      end
     end
 
     private
-    def digdag_url
+    def digdag_url(version=nil)
       url = ENV.fetch('TD_DIGDAG_URL', '').strip
       return url unless url.empty?
       user = Config.read['account.user']
@@ -131,7 +148,7 @@ module TreasureData
         return digdag_base_url
       end
       query = URI.encode_www_form('user' => user)
-      "#{digdag_base_url}?#{query}"
+      "#{digdag_base_url(version)}?#{query}"
     end
 
     private
@@ -406,7 +423,7 @@ EOF
       end
     end
 
-    def download_digdag
+    def download_digdag(version=nil)
       require 'net/http'
       require 'openssl'
 
@@ -422,7 +439,7 @@ EOF
         status = nil
         download_path = File.join(download_dir, 'digdag')
         File.open(download_path, 'wb') do |file|
-          status = Updater.stream_fetch(digdag_url, file) {
+          status = Updater.stream_fetch(digdag_url(version), file) {
             indicator.update
           }
         end

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.15.2'
+  TOOLBELT_VERSION = '0.15.3'
 end

data/td.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 1.8.0"
-  gem.add_dependency "td-client", "~> 0.8.85"
+  gem.add_dependency "td-client", ">= 0.8.85", "< 2"
   gem.add_dependency "td-logger", ">= 0.3.21", "< 2"
-  gem.add_dependency "rubyzip", "~> 1.1.7"
+  gem.add_dependency "rubyzip", ">= 1.2.1"
   gem.add_dependency "zip-zip", "~> 0.3"
   gem.add_dependency "ruby-progressbar", "~> 1.7"
   gem.add_development_dependency "rake"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.15.3
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-29 00:00:00.000000000 Z
+date: 2017-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -70,16 +70,22 @@ dependencies:
   name: td-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.85
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.85
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
@@ -104,16 +110,16 @@ dependencies:
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.7
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.7
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: zip-zip
   requirement: !ruby/object:Gem::Requirement
@@ -303,7 +309,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing