td 0.15.0 → 0.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +3 -5
- data/ChangeLog +11 -0
- data/bin/td +2 -4
- data/lib/td/command/export.rb +28 -0
- data/lib/td/command/job.rb +2 -14
- data/lib/td/command/list.rb +3 -0
- data/lib/td/command/query.rb +4 -4
- data/lib/td/command/sched.rb +2 -2
- data/lib/td/command/schema.rb +12 -49
- data/lib/td/command/workflow.rb +1 -1
- data/lib/td/version.rb +1 -1
- data/spec/td/command/export_spec.rb +53 -0
- data/spec/td/command/query_spec.rb +76 -12
- data/spec/td/command/schema_spec.rb +81 -0
- data/spec/td/command/workflow_spec.rb +14 -0
- data/td.gemspec +8 -8
- metadata +32 -54
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d67f821df06a365e2fc2b35febb963900e683f0a
|
4
|
+
data.tar.gz: 5af9e2025dbb5a756e0e2f8b4d53b105613165cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 98afa7ab296af4840874998df5dc55a940083341d6bb83768b3353b33774ea8948e95fd21b4794a8c171a8e4f7fc92939a0328f44dd0eb546f8747f1ab00ab83
|
7
|
+
data.tar.gz: b6a0b37882e836f90f90cd62da7d7adfb438af1569e2eca7702df483a5ca843bf079338d020c0447372b4657eaa8fd922c2f8d94e2c5eb1253218712877a4816
|
data/.travis.yml
CHANGED
data/ChangeLog
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
== 2016-11-29 version 0.15.2
|
2
|
+
|
3
|
+
* Deprecate Pig query engine #172
|
4
|
+
* Add export:result and export:table #175
|
5
|
+
|
6
|
+
== 2016-10-20 version 0.15.1
|
7
|
+
|
8
|
+
* Add --wait[=SECONDS] to query subcommand #171
|
9
|
+
* Drop Ruby 2.0.0 support
|
10
|
+
* Show sql_alias if exist #170
|
11
|
+
|
1
12
|
== 2016-09-06 version 0.15.0
|
2
13
|
|
3
14
|
* Add `td workflows` command. #167
|
data/bin/td
CHANGED
@@ -1,10 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# -*- coding: utf-8 -*-
|
3
3
|
|
4
|
-
require '
|
5
|
-
|
6
|
-
here = File.dirname(__FILE__)
|
7
|
-
$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
|
4
|
+
require 'td-client'
|
5
|
+
$LOAD_PATH << File.expand_path('../../lib', __FILE__)
|
8
6
|
|
9
7
|
# disable the updater for the td gem
|
10
8
|
require 'td/updater'
|
data/lib/td/command/export.rb
CHANGED
@@ -4,6 +4,34 @@ module Command
|
|
4
4
|
SUPPORTED_FORMATS = %W[json.gz line-json.gz tsv.gz]
|
5
5
|
SUPPORTED_ENCRYPT_METHOD = %W[s3]
|
6
6
|
|
7
|
+
def export_result(op)
|
8
|
+
wait = false
|
9
|
+
|
10
|
+
op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
|
11
|
+
wait = b
|
12
|
+
}
|
13
|
+
|
14
|
+
target_job_id, result = op.cmd_parse
|
15
|
+
|
16
|
+
client = get_ssl_client
|
17
|
+
|
18
|
+
opts = {}
|
19
|
+
opts['result'] = result
|
20
|
+
job = client.result_export(target_job_id, opts)
|
21
|
+
|
22
|
+
$stderr.puts "result export job #{job.job_id} is queued."
|
23
|
+
$stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
|
24
|
+
|
25
|
+
if wait && !job.finished?
|
26
|
+
wait_job(job)
|
27
|
+
$stdout.puts "status : #{job.status}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def export_table(op)
|
32
|
+
table_export(op)
|
33
|
+
end
|
34
|
+
|
7
35
|
def table_export(op)
|
8
36
|
from = nil
|
9
37
|
to = nil
|
data/lib/td/command/job.rb
CHANGED
@@ -225,26 +225,14 @@ private
|
|
225
225
|
$stdout.puts "\rUse '-v' option to show detailed messages." + " " * 20 unless verbose
|
226
226
|
end
|
227
227
|
|
228
|
-
def wait_job(job, first_call = false)
|
228
|
+
def wait_job(job, first_call = false, wait = nil)
|
229
229
|
$stderr.puts "queued..."
|
230
230
|
|
231
231
|
cmdout_lines = 0
|
232
232
|
stderr_lines = 0
|
233
233
|
max_error_counts = JOB_WAIT_MAX_RETRY_COUNT_ON_NETWORK_ERROR
|
234
234
|
|
235
|
-
|
236
|
-
first_call = false
|
237
|
-
begin
|
238
|
-
sleep 2
|
239
|
-
job.update_status!
|
240
|
-
rescue Timeout::Error, SystemCallError, EOFError, SocketError
|
241
|
-
if max_error_counts <= 0
|
242
|
-
raise
|
243
|
-
end
|
244
|
-
max_error_counts -= 1
|
245
|
-
retry
|
246
|
-
end
|
247
|
-
|
235
|
+
job.wait(wait, detail: true, verbose: true) do
|
248
236
|
cmdout = job.debug['cmdout'].to_s.split("\n")[cmdout_lines..-1] || []
|
249
237
|
stderr = job.debug['stderr'].to_s.split("\n")[stderr_lines..-1] || []
|
250
238
|
(cmdout + stderr).each {|line|
|
data/lib/td/command/list.rb
CHANGED
@@ -229,6 +229,9 @@ module List
|
|
229
229
|
add_list 'db:create', %w[db], 'Create a database', ['db:create example_db']
|
230
230
|
add_list 'db:delete', %w[db], 'Delete a database', ['db:delete example_db']
|
231
231
|
|
232
|
+
add_list 'export:result', %w[id, name], 'Dump logs of a target query to the specified place', ['export:result target_job_id result']
|
233
|
+
add_list 'export:table', %w[db table], 'Dump logs in a table to the specified storage', ['export:table example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY']
|
234
|
+
|
232
235
|
add_list 'table:list', %w[db?], 'Show list of tables', ['table:list', 'table:list example_db', 'tables']
|
233
236
|
add_list 'table:show', %w[db table], 'Describe information of a table', ['table example_db table1']
|
234
237
|
add_list 'table:create', %w[db table], 'Create a table', ['table:create example_db table1']
|
data/lib/td/command/query.rb
CHANGED
@@ -24,7 +24,7 @@ module Command
|
|
24
24
|
op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
|
25
25
|
db_name = s
|
26
26
|
}
|
27
|
-
op.on('-w', '--wait', 'wait for finishing the job',
|
27
|
+
op.on('-w', '--wait[=SECONDS]', 'wait for finishing the job (for seconds)', Integer) {|b|
|
28
28
|
wait = b
|
29
29
|
}
|
30
30
|
op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b|
|
@@ -63,7 +63,7 @@ module Command
|
|
63
63
|
op.on('-q', '--query PATH', 'use file instead of inline query') {|s|
|
64
64
|
query = File.open(s) { |f| f.read.strip }
|
65
65
|
}
|
66
|
-
op.on('-T', '--type TYPE', 'set query type (hive,
|
66
|
+
op.on('-T', '--type TYPE', 'set query type (hive, presto)') {|s|
|
67
67
|
type = s.to_sym
|
68
68
|
}
|
69
69
|
op.on('--sampling DENOMINATOR', 'OBSOLETE - enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|
|
@@ -145,8 +145,8 @@ module Command
|
|
145
145
|
$stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
|
146
146
|
#puts "See #{job.url} to see the progress."
|
147
147
|
|
148
|
-
if wait
|
149
|
-
wait_job(job, true)
|
148
|
+
if wait != false # `wait==nil` means `--wait` is specified
|
149
|
+
wait_job(job, true, wait)
|
150
150
|
$stdout.puts "Status : #{job.status}"
|
151
151
|
if job.success? && !exclude
|
152
152
|
begin
|
data/lib/td/command/sched.rb
CHANGED
@@ -76,7 +76,7 @@ module Command
|
|
76
76
|
op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
|
77
77
|
retry_limit = i
|
78
78
|
}
|
79
|
-
op.on('-T', '--type TYPE', 'set query type (hive
|
79
|
+
op.on('-T', '--type TYPE', 'set query type (hive)') {|s|
|
80
80
|
type = s
|
81
81
|
}
|
82
82
|
|
@@ -190,7 +190,7 @@ module Command
|
|
190
190
|
op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
|
191
191
|
retry_limit = i
|
192
192
|
}
|
193
|
-
op.on('-T', '--type TYPE', 'set query type (hive
|
193
|
+
op.on('-T', '--type TYPE', 'set query type (hive)') {|s|
|
194
194
|
type = s
|
195
195
|
}
|
196
196
|
|
data/lib/td/command/schema.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
|
-
module TreasureData
|
3
|
-
module Command
|
1
|
+
module TreasureData::Command
|
4
2
|
|
5
3
|
def schema_show(op)
|
6
4
|
db_name, table_name = op.cmd_parse
|
@@ -10,33 +8,35 @@ module Command
|
|
10
8
|
|
11
9
|
$stdout.puts "#{db_name}.#{table_name} ("
|
12
10
|
table.schema.fields.each {|f|
|
13
|
-
|
11
|
+
if f.sql_alias
|
12
|
+
$stdout.puts " #{f.name}:#{f.type}@#{f.sql_alias}"
|
13
|
+
else
|
14
|
+
$stdout.puts " #{f.name}:#{f.type}"
|
15
|
+
end
|
14
16
|
}
|
15
17
|
$stdout.puts ")"
|
16
18
|
end
|
17
19
|
|
18
20
|
def schema_set(op)
|
19
21
|
db_name, table_name, *columns = op.cmd_parse
|
20
|
-
schema = parse_columns(columns)
|
21
22
|
|
22
23
|
client = get_client
|
23
24
|
table = get_table(client, db_name, table_name)
|
24
25
|
|
25
|
-
|
26
|
+
schema = TreasureData::Schema.parse(columns)
|
27
|
+
client.update_schema(db_name, table_name, schema)
|
26
28
|
|
27
29
|
$stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
|
28
30
|
end
|
29
31
|
|
30
32
|
def schema_add(op)
|
31
33
|
db_name, table_name, *columns = op.cmd_parse
|
32
|
-
schema = parse_columns(columns)
|
33
34
|
|
34
35
|
client = get_client
|
35
36
|
table = get_table(client, db_name, table_name)
|
36
37
|
|
37
|
-
schema = table.schema.merge(
|
38
|
-
|
39
|
-
client.update_schema(table.db_name, table.name, schema)
|
38
|
+
schema = table.schema.merge(TreasureData::Schema.parse(columns))
|
39
|
+
client.update_schema(db_name, table_name, schema)
|
40
40
|
|
41
41
|
$stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
|
42
42
|
end
|
@@ -50,51 +50,14 @@ module Command
|
|
50
50
|
schema = table.schema
|
51
51
|
|
52
52
|
columns.each {|col|
|
53
|
-
|
54
|
-
schema.fields.delete_if {|f|
|
55
|
-
f.name == col && deleted = true
|
56
|
-
}
|
57
|
-
unless deleted
|
53
|
+
unless schema.fields.reject!{|f| f.name == col }
|
58
54
|
$stderr.puts "Column name '#{col}' does not exist."
|
59
55
|
exit 1
|
60
56
|
end
|
61
57
|
}
|
62
58
|
|
63
|
-
client.update_schema(
|
59
|
+
client.update_schema(db_name, table_name, schema)
|
64
60
|
|
65
61
|
$stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
|
66
62
|
end
|
67
|
-
|
68
|
-
private
|
69
|
-
def parse_columns(columns)
|
70
|
-
schema = Schema.new
|
71
|
-
|
72
|
-
columns.each {|column|
|
73
|
-
name, type = column.split(':',2)
|
74
|
-
name = name.to_s
|
75
|
-
type = type.to_s
|
76
|
-
|
77
|
-
begin
|
78
|
-
API.validate_column_name(name)
|
79
|
-
rescue ParameterValidationError => e
|
80
|
-
raise ParameterConfigurationError, e
|
81
|
-
end
|
82
|
-
#type = API.normalize_type_name(type)
|
83
|
-
|
84
|
-
if schema.fields.find {|f| f.name == name }
|
85
|
-
$stderr.puts "Column name '#{name}' is duplicated."
|
86
|
-
exit 1
|
87
|
-
end
|
88
|
-
schema.add_field(name, type)
|
89
|
-
|
90
|
-
if name == 'v' || name == 'time'
|
91
|
-
$stderr.puts "Column name '#{name}' is reserved."
|
92
|
-
exit 1
|
93
|
-
end
|
94
|
-
}
|
95
|
-
|
96
|
-
schema
|
97
|
-
end
|
98
63
|
end
|
99
|
-
end
|
100
|
-
|
data/lib/td/command/workflow.rb
CHANGED
@@ -128,7 +128,7 @@ module TreasureData
|
|
128
128
|
return url unless url.empty?
|
129
129
|
user = Config.read['account.user']
|
130
130
|
if user.nil? or user.strip.empty?
|
131
|
-
|
131
|
+
return digdag_base_url
|
132
132
|
end
|
133
133
|
query = URI.encode_www_form('user' => user)
|
134
134
|
"#{digdag_base_url}?#{query}"
|
data/lib/td/version.rb
CHANGED
@@ -101,5 +101,58 @@ module TreasureData::Command
|
|
101
101
|
}.to_not raise_exception
|
102
102
|
end
|
103
103
|
end
|
104
|
+
|
105
|
+
describe '#export_table' do
|
106
|
+
let(:option) {
|
107
|
+
List::CommandParser.new("export:table", ["db_name", "table_name"], [], nil, option_list, true)
|
108
|
+
}
|
109
|
+
let(:option_list) { [database, table, "-b", bucket, "-p", path, "-k", key, "-s", pass, "-F", format] }
|
110
|
+
let(:database) { 'database' }
|
111
|
+
let(:table) { 'table' }
|
112
|
+
let(:bucket) { 'bucket' }
|
113
|
+
let(:path) { 'path' }
|
114
|
+
let(:key) { 'key' }
|
115
|
+
let(:pass) { 'pass' }
|
116
|
+
let(:format) { 'tsv.gz' }
|
117
|
+
let(:job_id) { 111 }
|
118
|
+
|
119
|
+
before do
|
120
|
+
client = double(:client)
|
121
|
+
job = double(:job, job_id: job_id)
|
122
|
+
allow(client).to receive(:export).and_return(job)
|
123
|
+
table = double(:table)
|
124
|
+
|
125
|
+
allow(command).to receive(:get_client).and_return(client)
|
126
|
+
allow(command).to receive(:get_table).and_return(table)
|
127
|
+
end
|
128
|
+
|
129
|
+
it 'export table successfully like #table_export' do
|
130
|
+
expect {
|
131
|
+
command.table_export(option)
|
132
|
+
}.to_not raise_exception
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe '#export_result' do
|
137
|
+
let(:option) {
|
138
|
+
List::CommandParser.new("export:result", ["target_job_id", "result_url"], [], nil, option_list, true)
|
139
|
+
}
|
140
|
+
let(:option_list) { [110, 'mysql://user:pass@host.com/database/table'] }
|
141
|
+
let(:job_id) { 111 }
|
142
|
+
|
143
|
+
before do
|
144
|
+
client = double(:client)
|
145
|
+
job = double(:job, job_id: job_id)
|
146
|
+
allow(client).to receive(:result_export).and_return(job)
|
147
|
+
|
148
|
+
allow(command).to receive(:get_client).and_return(client)
|
149
|
+
end
|
150
|
+
|
151
|
+
it 'export result successfully' do
|
152
|
+
expect {
|
153
|
+
command.export_result(option)
|
154
|
+
}.not_to raise_exception
|
155
|
+
end
|
156
|
+
end
|
104
157
|
end
|
105
158
|
end
|
@@ -23,20 +23,84 @@ module TreasureData::Command
|
|
23
23
|
expect(client).to receive(:database).with('sample_datasets')
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
26
|
+
describe 'domain key' do
|
27
|
+
it 'accepts --domain-key' do
|
28
|
+
expect(client).to receive(:query).
|
29
|
+
with("sample_datasets", "SELECT 1;", nil, nil, nil, {"domain_key"=>"hoge"}).
|
30
|
+
and_return(job)
|
31
|
+
op = List::CommandParser.new("query", %w[query], %w[], nil, ['--domain-key=hoge', '-dsample_datasets', 'SELECT 1;'], false)
|
32
|
+
command.query(op)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'raises error if --domain-key is duplicated' do
|
36
|
+
expect(client).to receive(:query).
|
37
|
+
with("sample_datasets", "SELECT 1;", nil, nil, nil, {"domain_key"=>"hoge"}).
|
38
|
+
and_raise(::TreasureData::AlreadyExistsError.new('Query failed: domain_key has already been taken'))
|
39
|
+
op = List::CommandParser.new("query", %w[query], %w[], nil, ['--domain-key=hoge', '-dsample_datasets', 'SELECT 1;'], false)
|
40
|
+
expect{ command.query(op) }.to raise_error(TreasureData::AlreadyExistsError)
|
41
|
+
end
|
32
42
|
end
|
33
43
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
44
|
+
describe 'wait' do
|
45
|
+
let (:job_id){ 123 }
|
46
|
+
let :job do
|
47
|
+
obj = TreasureData::Job.new(client, job_id, 'presto', 'SELECT 1;')
|
48
|
+
allow(obj).to receive(:debug).and_return(double.as_null_object)
|
49
|
+
allow(obj).to receive(:sleep){|arg|@now += arg}
|
50
|
+
obj
|
51
|
+
end
|
52
|
+
before do
|
53
|
+
expect(client).to receive(:query).
|
54
|
+
with("sample_datasets", "SELECT 1;", nil, nil, nil, {}).
|
55
|
+
and_return(job)
|
56
|
+
@now = 1_400_000_000
|
57
|
+
allow(Time).to receive(:now){ @now }
|
58
|
+
allow(command).to receive(:show_result_with_retry)
|
59
|
+
end
|
60
|
+
context 'success' do
|
61
|
+
before do
|
62
|
+
status = nil
|
63
|
+
allow(client).to receive_message_chain(:api, :show_job).with(job_id) do
|
64
|
+
if @now < 1_400_000_010
|
65
|
+
status = TreasureData::Job::STATUS_RUNNING
|
66
|
+
else
|
67
|
+
status = TreasureData::Job::STATUS_SUCCESS
|
68
|
+
end
|
69
|
+
nil
|
70
|
+
end
|
71
|
+
allow(client).to receive(:job_status).with(job_id){ status }
|
72
|
+
end
|
73
|
+
it 'works with --wait' do
|
74
|
+
op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait', '-dsample_datasets', 'SELECT 1;'], false)
|
75
|
+
expect(command.query(op)).to be_nil
|
76
|
+
end
|
77
|
+
it 'works with --wait=360' do
|
78
|
+
op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait=360', '-dsample_datasets', 'SELECT 1;'], false)
|
79
|
+
expect(command.query(op)).to be_nil
|
80
|
+
end
|
81
|
+
end
|
82
|
+
context 'temporary failure' do
|
83
|
+
before do
|
84
|
+
status = nil
|
85
|
+
allow(client).to receive_message_chain(:api, :show_job).with(job_id) do
|
86
|
+
if @now < 1_400_000_010
|
87
|
+
status = TreasureData::Job::STATUS_RUNNING
|
88
|
+
else
|
89
|
+
status = TreasureData::Job::STATUS_SUCCESS
|
90
|
+
end
|
91
|
+
nil
|
92
|
+
end
|
93
|
+
allow(client).to receive(:job_status).with(job_id){ status }
|
94
|
+
end
|
95
|
+
it 'works with --wait' do
|
96
|
+
op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait', '-dsample_datasets', 'SELECT 1;'], false)
|
97
|
+
expect(command.query(op)).to be_nil
|
98
|
+
end
|
99
|
+
it 'works with --wait=360' do
|
100
|
+
op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait=360', '-dsample_datasets', 'SELECT 1;'], false)
|
101
|
+
expect(command.query(op)).to be_nil
|
102
|
+
end
|
103
|
+
end
|
40
104
|
end
|
41
105
|
end
|
42
106
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'td/command/common'
|
3
|
+
require 'td/config'
|
4
|
+
require 'td/command/list'
|
5
|
+
require 'td/command/schema'
|
6
|
+
require 'td/client/model'
|
7
|
+
require 'time'
|
8
|
+
|
9
|
+
describe TreasureData::Command do
|
10
|
+
let(:client){ double('client') }
|
11
|
+
let(:table){ double('table', schema: schema) }
|
12
|
+
let(:db_name){ "database" }
|
13
|
+
let(:table_name){ "table" }
|
14
|
+
let(:command) { Class.new { include TreasureData::Command }.new }
|
15
|
+
let(:columns){ ["foo:int", "BAR\u3070\u30FC:string@bar", "baz:baz!:array<double>@baz"] }
|
16
|
+
let(:schema){ TreasureData::Schema.parse(columns) }
|
17
|
+
let(:stdout){ StringIO.new }
|
18
|
+
before do
|
19
|
+
allow(command).to receive(:get_client).and_return(client)
|
20
|
+
allow(command).to receive(:get_table).with(client, db_name, table_name).and_return(table)
|
21
|
+
allow(table).to receive(:get_table).with(client, db_name, table_name).and_return(table)
|
22
|
+
allow($stdout).to receive(:puts){|arg| stdout.puts(*arg) }
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'schema_show' do
|
26
|
+
let(:op){ double('op', cmd_parse: [db_name, table_name]) }
|
27
|
+
it 'puts $stdout the schema' do
|
28
|
+
command.schema_show(op)
|
29
|
+
expect(stdout.string).to eq <<eom
|
30
|
+
database.table (
|
31
|
+
foo:int
|
32
|
+
BAR\u3070\u30FC:string@bar
|
33
|
+
baz:baz!:array<double>@baz
|
34
|
+
)
|
35
|
+
eom
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe 'schema_set' do
|
40
|
+
let(:op){ double('op', cmd_parse: [db_name, table_name, *columns]) }
|
41
|
+
it 'calls client.update_schema' do
|
42
|
+
allow($stderr).to receive(:puts)
|
43
|
+
expect(client).to receive(:update_schema) do |x, y, z|
|
44
|
+
expect(x).to eq db_name
|
45
|
+
expect(y).to eq table_name
|
46
|
+
expect(z.to_json).to eq schema.to_json
|
47
|
+
end
|
48
|
+
command.schema_set(op)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
describe 'schema_add' do
|
53
|
+
let(:columns2){ ["foo2:int", "BAR\u30702:string@bar2", "baz:baz!2:array<double>@baz2"] }
|
54
|
+
let(:schema2){ TreasureData::Schema.parse(columns+columns2) }
|
55
|
+
let(:op){ double('op', cmd_parse: [db_name, table_name, *columns2]) }
|
56
|
+
it 'calls client.update_schema' do
|
57
|
+
allow($stderr).to receive(:puts)
|
58
|
+
expect(client).to receive(:update_schema) do |x, y, z|
|
59
|
+
expect(x).to eq db_name
|
60
|
+
expect(y).to eq table_name
|
61
|
+
expect(z.to_json).to eq schema2.to_json
|
62
|
+
end
|
63
|
+
command.schema_add(op)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe 'schema_remove' do
|
68
|
+
let(:op){ double('op', cmd_parse: [db_name, table_name, "foo"]) }
|
69
|
+
let(:columns2){ ["BAR\u3070\u30FC:string@bar", "baz:baz!:array<double>@baz"] }
|
70
|
+
let(:schema2){ TreasureData::Schema.parse(columns2) }
|
71
|
+
it 'calls client.update_schema' do
|
72
|
+
allow($stderr).to receive(:puts)
|
73
|
+
expect(client).to receive(:update_schema) do |x, y, z|
|
74
|
+
expect(x).to eq db_name
|
75
|
+
expect(y).to eq table_name
|
76
|
+
expect(z.to_json).to eq schema.to_json
|
77
|
+
end
|
78
|
+
command.schema_remove(op)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -240,6 +240,20 @@ EOF
|
|
240
240
|
}
|
241
241
|
end
|
242
242
|
|
243
|
+
it 'downloads digdag even if there is no user' do
|
244
|
+
with_env('TD_WF_JAVA', 'java') {
|
245
|
+
allow(TreasureData::Config).to receive(:read) {{}}
|
246
|
+
allow(TreasureData::Updater).to receive(:stream_fetch).and_call_original
|
247
|
+
allow($stdin).to receive(:gets) { 'Y' }
|
248
|
+
status = command.workflow(empty_option, capture_output=true)
|
249
|
+
expect(status).to be 0
|
250
|
+
expect(stdout_io.string).to include 'Downloading workflow module'
|
251
|
+
expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag', 'digdag'))
|
252
|
+
expect(TreasureData::Updater).to have_received(:stream_fetch).with(
|
253
|
+
'http://toolbelt.treasure-data.com/digdag', instance_of(File))
|
254
|
+
}
|
255
|
+
end
|
256
|
+
|
243
257
|
it 'reinstalls cleanly after reset' do
|
244
258
|
skip 'Requires 64 bit OS or system java' unless (TreasureData::Helpers::on_64bit_os? or java_available?)
|
245
259
|
|
data/td.gemspec
CHANGED
@@ -15,19 +15,19 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
16
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
17
|
gem.require_paths = ['lib']
|
18
|
-
gem.required_ruby_version = '>= 1
|
18
|
+
gem.required_ruby_version = '>= 2.1'
|
19
19
|
|
20
|
-
gem.add_dependency "msgpack"
|
20
|
+
gem.add_dependency "msgpack"
|
21
21
|
gem.add_dependency "yajl-ruby", "~> 1.1"
|
22
22
|
gem.add_dependency "hirb", ">= 0.4.5"
|
23
23
|
gem.add_dependency "parallel", "~> 1.8.0"
|
24
|
-
gem.add_dependency "td-client", "~> 0.8.
|
25
|
-
gem.add_dependency "td-logger", "
|
24
|
+
gem.add_dependency "td-client", "~> 0.8.85"
|
25
|
+
gem.add_dependency "td-logger", ">= 0.3.21", "< 2"
|
26
26
|
gem.add_dependency "rubyzip", "~> 1.1.7"
|
27
27
|
gem.add_dependency "zip-zip", "~> 0.3"
|
28
|
-
gem.add_dependency "ruby-progressbar", "~> 1.7
|
29
|
-
gem.add_development_dependency "rake"
|
30
|
-
gem.add_development_dependency "rspec"
|
31
|
-
gem.add_development_dependency "simplecov"
|
28
|
+
gem.add_dependency "ruby-progressbar", "~> 1.7"
|
29
|
+
gem.add_development_dependency "rake"
|
30
|
+
gem.add_development_dependency "rspec"
|
31
|
+
gem.add_development_dependency "simplecov"
|
32
32
|
gem.add_development_dependency 'coveralls'
|
33
33
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.15.
|
4
|
+
version: 0.15.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Treasure Data, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -16,44 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0
|
20
|
-
- - "!="
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 0.5.0
|
23
|
-
- - "!="
|
24
|
-
- !ruby/object:Gem::Version
|
25
|
-
version: 0.5.1
|
26
|
-
- - "!="
|
27
|
-
- !ruby/object:Gem::Version
|
28
|
-
version: 0.5.2
|
29
|
-
- - "!="
|
30
|
-
- !ruby/object:Gem::Version
|
31
|
-
version: 0.5.3
|
32
|
-
- - "<"
|
33
|
-
- !ruby/object:Gem::Version
|
34
|
-
version: 0.8.0
|
19
|
+
version: '0'
|
35
20
|
type: :runtime
|
36
21
|
prerelease: false
|
37
22
|
version_requirements: !ruby/object:Gem::Requirement
|
38
23
|
requirements:
|
39
24
|
- - ">="
|
40
25
|
- !ruby/object:Gem::Version
|
41
|
-
version: 0
|
42
|
-
- - "!="
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
-
version: 0.5.0
|
45
|
-
- - "!="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: 0.5.1
|
48
|
-
- - "!="
|
49
|
-
- !ruby/object:Gem::Version
|
50
|
-
version: 0.5.2
|
51
|
-
- - "!="
|
52
|
-
- !ruby/object:Gem::Version
|
53
|
-
version: 0.5.3
|
54
|
-
- - "<"
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: 0.8.0
|
26
|
+
version: '0'
|
57
27
|
- !ruby/object:Gem::Dependency
|
58
28
|
name: yajl-ruby
|
59
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -102,28 +72,34 @@ dependencies:
|
|
102
72
|
requirements:
|
103
73
|
- - "~>"
|
104
74
|
- !ruby/object:Gem::Version
|
105
|
-
version: 0.8.
|
75
|
+
version: 0.8.85
|
106
76
|
type: :runtime
|
107
77
|
prerelease: false
|
108
78
|
version_requirements: !ruby/object:Gem::Requirement
|
109
79
|
requirements:
|
110
80
|
- - "~>"
|
111
81
|
- !ruby/object:Gem::Version
|
112
|
-
version: 0.8.
|
82
|
+
version: 0.8.85
|
113
83
|
- !ruby/object:Gem::Dependency
|
114
84
|
name: td-logger
|
115
85
|
requirement: !ruby/object:Gem::Requirement
|
116
86
|
requirements:
|
117
|
-
- - "
|
87
|
+
- - ">="
|
118
88
|
- !ruby/object:Gem::Version
|
119
89
|
version: 0.3.21
|
90
|
+
- - "<"
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: '2'
|
120
93
|
type: :runtime
|
121
94
|
prerelease: false
|
122
95
|
version_requirements: !ruby/object:Gem::Requirement
|
123
96
|
requirements:
|
124
|
-
- - "
|
97
|
+
- - ">="
|
125
98
|
- !ruby/object:Gem::Version
|
126
99
|
version: 0.3.21
|
100
|
+
- - "<"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '2'
|
127
103
|
- !ruby/object:Gem::Dependency
|
128
104
|
name: rubyzip
|
129
105
|
requirement: !ruby/object:Gem::Requirement
|
@@ -158,56 +134,56 @@ dependencies:
|
|
158
134
|
requirements:
|
159
135
|
- - "~>"
|
160
136
|
- !ruby/object:Gem::Version
|
161
|
-
version: 1.7
|
137
|
+
version: '1.7'
|
162
138
|
type: :runtime
|
163
139
|
prerelease: false
|
164
140
|
version_requirements: !ruby/object:Gem::Requirement
|
165
141
|
requirements:
|
166
142
|
- - "~>"
|
167
143
|
- !ruby/object:Gem::Version
|
168
|
-
version: 1.7
|
144
|
+
version: '1.7'
|
169
145
|
- !ruby/object:Gem::Dependency
|
170
146
|
name: rake
|
171
147
|
requirement: !ruby/object:Gem::Requirement
|
172
148
|
requirements:
|
173
|
-
- - "
|
149
|
+
- - ">="
|
174
150
|
- !ruby/object:Gem::Version
|
175
|
-
version: '0
|
151
|
+
version: '0'
|
176
152
|
type: :development
|
177
153
|
prerelease: false
|
178
154
|
version_requirements: !ruby/object:Gem::Requirement
|
179
155
|
requirements:
|
180
|
-
- - "
|
156
|
+
- - ">="
|
181
157
|
- !ruby/object:Gem::Version
|
182
|
-
version: '0
|
158
|
+
version: '0'
|
183
159
|
- !ruby/object:Gem::Dependency
|
184
160
|
name: rspec
|
185
161
|
requirement: !ruby/object:Gem::Requirement
|
186
162
|
requirements:
|
187
|
-
- - "
|
163
|
+
- - ">="
|
188
164
|
- !ruby/object:Gem::Version
|
189
|
-
version:
|
165
|
+
version: '0'
|
190
166
|
type: :development
|
191
167
|
prerelease: false
|
192
168
|
version_requirements: !ruby/object:Gem::Requirement
|
193
169
|
requirements:
|
194
|
-
- - "
|
170
|
+
- - ">="
|
195
171
|
- !ruby/object:Gem::Version
|
196
|
-
version:
|
172
|
+
version: '0'
|
197
173
|
- !ruby/object:Gem::Dependency
|
198
174
|
name: simplecov
|
199
175
|
requirement: !ruby/object:Gem::Requirement
|
200
176
|
requirements:
|
201
|
-
- - "
|
177
|
+
- - ">="
|
202
178
|
- !ruby/object:Gem::Version
|
203
|
-
version: 0
|
179
|
+
version: '0'
|
204
180
|
type: :development
|
205
181
|
prerelease: false
|
206
182
|
version_requirements: !ruby/object:Gem::Requirement
|
207
183
|
requirements:
|
208
|
-
- - "
|
184
|
+
- - ">="
|
209
185
|
- !ruby/object:Gem::Version
|
210
|
-
version: 0
|
186
|
+
version: '0'
|
211
187
|
- !ruby/object:Gem::Dependency
|
212
188
|
name: coveralls
|
213
189
|
requirement: !ruby/object:Gem::Requirement
|
@@ -293,6 +269,7 @@ files:
|
|
293
269
|
- spec/td/command/job_spec.rb
|
294
270
|
- spec/td/command/query_spec.rb
|
295
271
|
- spec/td/command/sched_spec.rb
|
272
|
+
- spec/td/command/schema_spec.rb
|
296
273
|
- spec/td/command/table_spec.rb
|
297
274
|
- spec/td/command/workflow_spec.rb
|
298
275
|
- spec/td/common_spec.rb
|
@@ -318,7 +295,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
318
295
|
requirements:
|
319
296
|
- - ">="
|
320
297
|
- !ruby/object:Gem::Version
|
321
|
-
version: '1
|
298
|
+
version: '2.1'
|
322
299
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
323
300
|
requirements:
|
324
301
|
- - ">="
|
@@ -326,7 +303,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
326
303
|
version: '0'
|
327
304
|
requirements: []
|
328
305
|
rubyforge_project:
|
329
|
-
rubygems_version: 2.5.
|
306
|
+
rubygems_version: 2.5.2
|
330
307
|
signing_key:
|
331
308
|
specification_version: 4
|
332
309
|
summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
|
@@ -344,6 +321,7 @@ test_files:
|
|
344
321
|
- spec/td/command/job_spec.rb
|
345
322
|
- spec/td/command/query_spec.rb
|
346
323
|
- spec/td/command/sched_spec.rb
|
324
|
+
- spec/td/command/schema_spec.rb
|
347
325
|
- spec/td/command/table_spec.rb
|
348
326
|
- spec/td/command/workflow_spec.rb
|
349
327
|
- spec/td/common_spec.rb
|