td 0.15.7 → 0.15.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7bb4903ebbe92100825f6fb8c768a4b43135e32e
-  data.tar.gz: 6c2fce67432e0d56d536971b5b75e164d88a8bc4
+  metadata.gz: a279860c461d45d5c1f7450e7b892ffcad60b5bd
+  data.tar.gz: c03240d59c09335f2dc2985011c79b73a00e6389
 SHA512:
-  metadata.gz: 39f2f0b420f772c670990e1303a037f626bf989c208a75103724fbc24359755335d68fc1623b1c8aaadbcca964280164aea34636cbde90eb653c298274bf6462
-  data.tar.gz: 0c4b3cc6037493a693d8d283160136bd2b7bda2f701aaab7705895322285bc388df348646245d96e214c5f338298904fbacca113bed86477945dccf9a7ae1126
+  metadata.gz: '09b2934ab370519543a1b422ec7afdefcc1a9e2ec6f1511e08f214d9bf440f14d785a45eabd4028bd43c2765d819f300d8180eacab98b76f83ab05dc36eca33b'
+  data.tar.gz: 989748a1001685d4dfa1ed47c046fe08e888377631171e9a52bd1e87aa50d66cd766b7ebb6367522c19dc648e64d5fbea11fe68ab055ef46a06e57b99a1c447e

data/.travis.yml

@@ -5,8 +5,9 @@ jdk:
 
 rvm:
   - 2.1.10
-  - 2.2.5
-  - 2.3.1
+  - 2.2.9
+  - 2.3.6
+  - 2.4.3
   - ruby-head
 
 gemfile:

data/ChangeLog

@@ -1,3 +1,10 @@
+== 2018-01-22 version 0.15.8
+
+* table:update to update table options #196
+* Avoid extra API call on job:list #199
+* td workflow supports non US region and Private Connect #197 #198
+* fix sched:history -p causes missing one record #195
+
 == 2017-10-24 version 0.15.7
 
 * Support workflow on aws tokyo #94

data/lib/td/command/job.rb

@@ -74,6 +74,7 @@ module Command
 
     rows = []
     jobs.each {|job|
+      job.auto_update_status = false
       start = job.start_at
       elapsed = Command.humanize_elapsed_time(start, job.end_at)
       cpu_time = Command.humanize_time(job.cpu_time, true)

data/lib/td/command/list.rb

@@ -243,6 +243,7 @@ module List
   add_list 'table:tail', %w[db table], 'Get recently imported logs', ['table:tail example_db table1', 'table:tail example_db table1 -n 30']
   add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', ['table:partial_delete example_db table1 --from 1341000000 --to 1341003600']
   add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days. Set to 0 to disable the expiration.', ['table:expire example_db table1 30']
+  add_list 'table:update', %w[db table], 'Update table options', ['table:update example_db table1 --include-v false']
 
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', ['bulk_import:list']
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', ['bulk_import:show']

data/lib/td/command/sched.rb

@@ -256,7 +256,7 @@ module Command
     client = get_client
 
     begin
-      history = client.history(name, skip, skip+max-1)
+      history = client.history(name, skip, skip+max)
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."

data/lib/td/command/table.rb

@@ -15,6 +15,7 @@ module Command
   KEY_NUM_LIMIT = 512
 
   def table_create(op)
+    params = {}
     type = nil
 
     op.on('-T', '--type TYPE', 'set table type (log)') {|s|
@@ -23,6 +24,15 @@ module Command
       end
       type = s.to_sym
     }
+    op.on('--expire-days DAYS', Integer, 'set table expire days') do |v|
+      if v < 0
+        $stderr.puts "Table expiration days must be greater or equal to 0."
+        return
+      end
+      params[:expire_days] = v
+    end
+    op.on('--include-v BOOLEAN', TrueClass, 'set include_v flag') {|v| params[:include_v] = v}
+    op.on('--detect-schema BOOLEAN', TrueClass, 'set detect schema flag') {|v| params[:detect_schema] = v}
     db_name, table_name = op.cmd_parse
 
     API.validate_table_name(table_name)
@@ -36,7 +46,7 @@ module Command
     client = get_client
 
     begin
-      client.create_log_table(db_name, table_name)
+      client.create_log_table(db_name, table_name, params)
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Database '#{db_name}' does not exist."
@@ -218,6 +228,10 @@ module Command
   end
 
   def table_show(op)
+    verbose = nil
+    op.on('-v', 'show more attributes', TrueClass) {|b|
+      verbose = b
+    }
     db_name, table_name = op.cmd_parse
 
     client = get_client
@@ -227,7 +241,10 @@ module Command
     $stdout.puts "Name        : #{table.db_name}.#{table.name}"
     $stdout.puts "Type        : #{table.type}"
     $stdout.puts "Count       : #{table.count}"
-    # p table.methods.each {|m| $stdout.puts m}
+    if verbose
+      $stdout.puts "Expire Days : #{table.expire_days}"
+      $stdout.puts "Include v   : #{table.include_v}"
+    end
     $stdout.puts "Schema      : ("
     table.schema.fields.each {|f|
       $stdout.puts "    #{f.name}:#{f.type}"
@@ -336,6 +353,25 @@ module Command
     end
   end
 
+  def table_update(op)
+    params = {}
+    op.on('--expire-days DAYS', Integer, 'set table expire days') do |v|
+      if v < 0
+        $stderr.puts "Table expiration days must be greater or equal to 0."
+        return
+      end
+      params[:expire_days] = v
+    end
+    op.on('--include-v BOOLEAN', TrueClass, 'set include_v flag') {|v| params[:include_v] = v}
+    op.on('--detect-schema BOOLEAN', TrueClass, 'set detect schema flag') {|v| params[:detect_schema] = v}
+    db_name, table_name = op.cmd_parse
+
+    client = get_client
+    res = client.update_table(db_name, table_name, params)
+
+    $stdout.puts res.inspect
+  end
+
   def table_expire(op)
     db_name, table_name, expire_days = op.cmd_parse
 

data/lib/td/command/workflow.rb

@@ -42,6 +42,7 @@ module TreasureData
               "secrets.td.apikey = #{apikey}"
           ].join($/) + $/)
           cmd << '-Dio.digdag.standards.td.secrets.enabled=false'
+          cmd << "-Dconfig.td.default_endpoint=#{Config.endpoint_domain}"
         else
           # Use the digdag td.conf plugin to configure wf api and apikey.
           env['TREASURE_DATA_CONFIG_PATH'] = Config.path

data/lib/td/config.rb

@@ -152,6 +152,10 @@ class Config
     @@endpoint = endpoint
   end
 
+  def self.endpoint_domain
+    (self.endpoint || 'api.treasuredata.com').sub(%r[https?://], '')
+  end
+
   def self.cl_endpoint
     @@cl_endpoint
   end
@@ -161,13 +165,17 @@ class Config
   end
 
   def self.workflow_endpoint
-    case self.endpoint.to_s.sub(%r[https?://], '')
-    when '', 'api.treasuredata.com'
+    case self.endpoint_domain
+    when 'api.treasuredata.com'
       'https://api-workflow.treasuredata.com'
+    when /\Aapi-([^.]*).connect.treasuredata.com\z/
+      "https://api-workflow-#{$1}.connect.treasuredata.com"
     when 'api-staging.treasuredata.com'
       'https://api-staging-workflow.treasuredata.com'
     when 'api.treasuredata.co.jp'
       'https://api-workflow.treasuredata.co.jp'
+    when /\Aapi-([^.]*).connect.treasuredata.co.jp\z/
+      "https://api-workflow-#{$1}.connect.treasuredata.co.jp"
     when 'api-staging.treasuredata.co.jp'
       'https://api-staging-workflow.treasuredata.co.jp'
     else

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.15.7'
+  TOOLBELT_VERSION = '0.15.8'
 end

data/spec/td/command/job_spec.rb

@@ -348,46 +348,39 @@ text
 
       let(:job_id) { "12345" }
 
-      let :job_class do
-        Struct.new(:job_id,
-                   :status,
-                   :type,
-                   :db_name,
-                   :priority,
-                   :retry_limit,
-                   :result_url,
-                   :query,
-                   :start_at,
-                   :end_at,
-                   :cpu_time,
-                   :result_size,
-                   :duration
-                  )
-      end
-
       let :start_at do
         Time.now
       end
 
+      let :client do
+        double('client')
+      end
+
       let :jobs do
-        [job_class.new(job_id,
-                       nil,
+        [TreasureData::Job.new(client,
+                       job_id,
                        :hive,
-                       "db_name",
-                       1,
-                       1,
-                       "test_url",
                        "test_qury",
-                       start_at,
-                       start_at + 10,
+                       nil,
+                       nil,
+                       nil,
+                       start_at.iso8601,
+                       (start_at + 10).iso8601,
                        1,
                        3,
-                       100
+                       nil,
+                       "test_url",
+                       nil,
+                       1,
+                       1,
+                       nil,
+                       "db_name",
+                       100,
+                       1,
                       )] * 3
       end
 
       before do
-        client = Object.new
         allow(client).to receive(:jobs).and_return(jobs)
         allow(command).to receive(:get_client).and_return(client)
       end

data/spec/td/command/table_spec.rb

@@ -180,6 +180,124 @@ module TreasureData::Command
       end
     end
 
+    describe '#table_create' do
+      let(:db_name)    { 'database' }
+      let(:table_name) { 'table1' }
+      let(:client)     { double('client') }
+      let(:command)    { Class.new { include TreasureData::Command }.new }
+
+      before do
+        allow(command).to receive(:get_client) { client }
+      end
+
+      context 'normal' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {})
+          command.table_create(option)
+        end
+      end
+
+      context 'with include_v' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--include-v'], false)
+        }
+
+        it 'raises Error' do
+          expect{ command.table_create(option) }.to raise_error(SystemExit)
+        end
+      end
+
+      context 'with include_v=true' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--include-v=true'], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {include_v: true})
+          expect{ command.table_create(option) }.not_to raise_error
+        end
+      end
+
+      context 'with include_v=false' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--include-v=false'], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {include_v: false})
+          expect{ command.table_create(option) }.not_to raise_error
+        end
+      end
+
+      context 'with detect_schema' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--detect-schema'], false)
+        }
+
+        it 'raises Error' do
+          expect{ command.table_create(option) }.to raise_error(SystemExit)
+        end
+      end
+
+      context 'with detect_schema=true' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--detect-schema=true'], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {detect_schema: true})
+          expect{ command.table_create(option) }.not_to raise_error
+        end
+      end
+
+      context 'with detect_schema=false' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--detect-schema=false'], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {detect_schema: false})
+          expect{ command.table_create(option) }.not_to raise_error
+        end
+      end
+
+      context 'with expire_days' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--expire-days'], false)
+        }
+
+        it 'raises Error' do
+          expect{ command.table_create(option) }.to raise_error(SystemExit)
+        end
+      end
+
+      context 'with expire_days=0' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--expire-days=0'], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {expire_days: 0})
+          expect{ command.table_create(option) }.not_to raise_error
+        end
+      end
+
+      context 'with expire_days=365' do
+        let(:option) {
+          List::CommandParser.new('table:create', %w(db_name table_name), [], false, [db_name, table_name, '--expire-days=365'], false)
+        }
+
+        it 'create table' do
+          expect(client).to receive(:create_log_table).with(db_name, table_name, {expire_days: 365})
+          expect{ command.table_create(option) }.not_to raise_error
+        end
+      end
+    end
+
     describe '#table_import' do
       let(:db_name)    { 'database' }
       let(:table_name) { 'table' }

data/spec/td/config_spec.rb

@@ -19,10 +19,18 @@ describe TreasureData::Config do
         it { is_expected.to eq 'https://api-workflow.treasuredata.com' }
       end
     end
+    context 'api-hoge.connect.treasuredata.com' do
+      let(:api_endpoint){ 'api-hoge.connect.treasuredata.com' }
+      it { is_expected.to eq 'https://api-workflow-hoge.connect.treasuredata.com' }
+    end
     context 'api.treasuredata.co.jp' do
       let(:api_endpoint){ 'api.treasuredata.co.jp' }
       it { is_expected.to eq 'https://api-workflow.treasuredata.co.jp' }
     end
+    context 'api-hoge.connect.treasuredata.co.jp' do
+      let(:api_endpoint){ 'api-hoge.connect.treasuredata.co.jp' }
+      it { is_expected.to eq 'https://api-workflow-hoge.connect.treasuredata.co.jp' }
+    end
     context 'api-staging.treasuredata.com' do
       let(:api_endpoint){ 'api-staging.treasuredata.com' }
       it { is_expected.to eq 'https://api-staging-workflow.treasuredata.com' }

data/td.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 1.8"
-  gem.add_dependency "td-client", ">= 1.0.4", "< 2"
+  gem.add_dependency "td-client", ">= 1.0.5", "< 2"
   gem.add_dependency "td-logger", ">= 0.3.21", "< 2"
   gem.add_dependency "rubyzip", ">= 1.2.1"
   gem.add_dependency "zip-zip", "~> 0.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.15.7
+  version: 0.15.8
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-24 00:00:00.000000000 Z
+date: 2018-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -72,7 +72,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.4
+        version: 1.0.5
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.4
+        version: 1.0.5
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'