td 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/ChangeLog +9 -0
- data/lib/td/command/export.rb +7 -0
- data/lib/td/command/job.rb +3 -0
- data/lib/td/command/query.rb +5 -0
- data/lib/td/command/runner.rb +1 -1
- data/lib/td/command/sched.rb +5 -1
- data/lib/td/updater.rb +18 -13
- data/lib/td/version.rb +1 -1
- data/spec/file_reader/filter_spec.rb +14 -14
- data/spec/file_reader/io_filter_spec.rb +7 -7
- data/spec/file_reader/line_reader_spec.rb +14 -14
- data/spec/file_reader/parsing_reader_spec.rb +5 -5
- data/spec/file_reader/shared_context.rb +1 -1
- data/spec/file_reader_spec.rb +39 -29
- data/spec/td/command/connector_spec.rb +21 -21
- data/spec/td/command/export_spec.rb +86 -0
- data/spec/td/command/import_spec.rb +11 -11
- data/spec/td/command/job_spec.rb +48 -48
- data/spec/td/command/query_spec.rb +42 -0
- data/spec/td/command/sched_spec.rb +6 -6
- data/spec/td/command/table_spec.rb +54 -54
- data/spec/td/common_spec.rb +21 -21
- data/spec/td/updater_spec.rb +8 -8
- data/spec/td/version_spec.rb +2 -2
- data/td.gemspec +3 -3
- metadata +12 -8
@@ -13,20 +13,20 @@ describe 'FileReader parsing readers' do
|
|
13
13
|
|
14
14
|
shared_examples_for 'forward basics' do
|
15
15
|
it 'forward returns one data' do
|
16
|
-
reader.forward.
|
16
|
+
expect(reader.forward).to eq(dataset[0])
|
17
17
|
end
|
18
18
|
|
19
19
|
it 'feeds all dataset' do
|
20
20
|
begin
|
21
21
|
i = 0
|
22
22
|
while line = reader.forward
|
23
|
-
line.
|
23
|
+
expect(line).to eq(dataset[i])
|
24
24
|
i += 1
|
25
25
|
end
|
26
26
|
rescue RSpec::Expectations::ExpectationNotMetError => e
|
27
27
|
fail
|
28
28
|
rescue => e
|
29
|
-
io.eof
|
29
|
+
expect(io.eof?).to be_truthy
|
30
30
|
end
|
31
31
|
end
|
32
32
|
end
|
@@ -46,7 +46,7 @@ describe 'FileReader parsing readers' do
|
|
46
46
|
|
47
47
|
it 'initialize' do
|
48
48
|
reader = FileReader::MessagePackParsingReader.new(io, error, {})
|
49
|
-
reader.
|
49
|
+
expect(reader).not_to be_nil
|
50
50
|
end
|
51
51
|
|
52
52
|
context 'after initialization' do
|
@@ -102,7 +102,7 @@ describe 'FileReader parsing readers' do
|
|
102
102
|
|
103
103
|
it "initialize #{format}" do
|
104
104
|
reader = FileReader::SeparatedValueParsingReader.new(io, error, opts)
|
105
|
-
reader.
|
105
|
+
expect(reader).not_to be_nil
|
106
106
|
end
|
107
107
|
|
108
108
|
context "after #{format} initialization" do
|
data/spec/file_reader_spec.rb
CHANGED
@@ -12,10 +12,20 @@ describe FileReader do
|
|
12
12
|
describe 'initialize' do
|
13
13
|
subject { FileReader.new }
|
14
14
|
|
15
|
-
|
16
|
-
|
15
|
+
describe '#parser_class' do
|
16
|
+
subject { super().parser_class }
|
17
|
+
it { is_expected.to be_nil }
|
18
|
+
end
|
19
|
+
|
20
|
+
describe '#opts' do
|
21
|
+
subject { super().opts }
|
22
|
+
it { is_expected.to be_empty }
|
23
|
+
end
|
17
24
|
[:delimiter_expr, :null_expr, :true_expr, :false_expr].each { |key|
|
18
|
-
|
25
|
+
describe '#default_opts' do
|
26
|
+
subject { super().default_opts }
|
27
|
+
it { is_expected.to have_key(key); }
|
28
|
+
end
|
19
29
|
}
|
20
30
|
end
|
21
31
|
|
@@ -26,36 +36,36 @@ describe FileReader do
|
|
26
36
|
describe 'set_format_template' do
|
27
37
|
it 'can set csv' do
|
28
38
|
reader.set_format_template('csv')
|
29
|
-
reader.instance_variable_get(:@format).
|
30
|
-
reader.opts.
|
39
|
+
expect(reader.instance_variable_get(:@format)).to eq('text')
|
40
|
+
expect(reader.opts).to include(:delimiter_expr => /,/)
|
31
41
|
end
|
32
42
|
|
33
43
|
it 'can set tsv' do
|
34
44
|
reader.set_format_template('tsv')
|
35
|
-
reader.instance_variable_get(:@format).
|
36
|
-
reader.opts.
|
45
|
+
expect(reader.instance_variable_get(:@format)).to eq('text')
|
46
|
+
expect(reader.opts).to include(:delimiter_expr => /\t/)
|
37
47
|
end
|
38
48
|
|
39
49
|
it 'can set apache' do
|
40
50
|
reader.set_format_template('apache')
|
41
|
-
reader.instance_variable_get(:@format).
|
42
|
-
reader.opts.
|
51
|
+
expect(reader.instance_variable_get(:@format)).to eq('apache')
|
52
|
+
expect(reader.opts).to include(:time_column => 'time')
|
43
53
|
end
|
44
54
|
|
45
55
|
it 'can set syslog' do
|
46
56
|
reader.set_format_template('syslog')
|
47
|
-
reader.instance_variable_get(:@format).
|
48
|
-
reader.opts.
|
57
|
+
expect(reader.instance_variable_get(:@format)).to eq('syslog')
|
58
|
+
expect(reader.opts).to include(:time_column => 'time')
|
49
59
|
end
|
50
60
|
|
51
61
|
it 'can set msgpack' do
|
52
62
|
reader.set_format_template('msgpack')
|
53
|
-
reader.instance_variable_get(:@format).
|
63
|
+
expect(reader.instance_variable_get(:@format)).to eq('msgpack')
|
54
64
|
end
|
55
65
|
|
56
66
|
it 'can set json' do
|
57
67
|
reader.set_format_template('json')
|
58
|
-
reader.instance_variable_get(:@format).
|
68
|
+
expect(reader.instance_variable_get(:@format)).to eq('json')
|
59
69
|
end
|
60
70
|
|
61
71
|
it 'raises when set unknown format' do
|
@@ -77,7 +87,7 @@ describe FileReader do
|
|
77
87
|
['-f', '--format'].each { |opt|
|
78
88
|
['csv', 'tsv', 'apache', 'syslog', 'msgpack', 'json'].each { |format|
|
79
89
|
it "#{opt} option with #{format}" do
|
80
|
-
reader.
|
90
|
+
expect(reader).to receive(:set_format_template).with(format)
|
81
91
|
parse_opt([opt, format]) { }
|
82
92
|
end
|
83
93
|
}
|
@@ -89,7 +99,7 @@ describe FileReader do
|
|
89
99
|
it "#{opt} option" do
|
90
100
|
columns = 'A,B,C'
|
91
101
|
parse_opt([opt, columns]) {
|
92
|
-
reader.opts.
|
102
|
+
expect(reader.opts).to include(:column_names => columns.split(','))
|
93
103
|
}
|
94
104
|
end
|
95
105
|
}
|
@@ -99,7 +109,7 @@ describe FileReader do
|
|
99
109
|
['-H', '--column-header'].each { |opt|
|
100
110
|
it "#{opt} option" do
|
101
111
|
parse_opt([opt]) {
|
102
|
-
reader.opts.
|
112
|
+
expect(reader.opts).to include(:column_header => true)
|
103
113
|
}
|
104
114
|
end
|
105
115
|
}
|
@@ -110,7 +120,7 @@ describe FileReader do
|
|
110
120
|
it "#{opt} option" do
|
111
121
|
pattern = '!'
|
112
122
|
parse_opt([opt, pattern]) {
|
113
|
-
reader.opts.
|
123
|
+
expect(reader.opts).to include(:delimiter_expr => Regexp.new(pattern))
|
114
124
|
}
|
115
125
|
end
|
116
126
|
}
|
@@ -120,7 +130,7 @@ describe FileReader do
|
|
120
130
|
it "--null REGEX option" do
|
121
131
|
pattern = 'null'
|
122
132
|
parse_opt(['--null', pattern]) {
|
123
|
-
reader.opts.
|
133
|
+
expect(reader.opts).to include(:null_expr => Regexp.new(pattern))
|
124
134
|
}
|
125
135
|
end
|
126
136
|
end
|
@@ -129,7 +139,7 @@ describe FileReader do
|
|
129
139
|
it "--true REGEX option" do
|
130
140
|
pattern = 'true'
|
131
141
|
parse_opt(['--true', pattern]) {
|
132
|
-
reader.opts.
|
142
|
+
expect(reader.opts).to include(:true_expr => Regexp.new(pattern))
|
133
143
|
}
|
134
144
|
end
|
135
145
|
end
|
@@ -138,7 +148,7 @@ describe FileReader do
|
|
138
148
|
it "--false REGEX option" do
|
139
149
|
pattern = 'false'
|
140
150
|
parse_opt(['--false', pattern]) {
|
141
|
-
reader.opts.
|
151
|
+
expect(reader.opts).to include(:false_expr => Regexp.new(pattern))
|
142
152
|
}
|
143
153
|
end
|
144
154
|
end
|
@@ -147,7 +157,7 @@ describe FileReader do
|
|
147
157
|
['-S', '--all-string'].each { |opt|
|
148
158
|
it "#{opt} option" do
|
149
159
|
parse_opt([opt]) {
|
150
|
-
reader.opts.
|
160
|
+
expect(reader.opts).to include(:all_string => true)
|
151
161
|
}
|
152
162
|
end
|
153
163
|
}
|
@@ -158,7 +168,7 @@ describe FileReader do
|
|
158
168
|
it "#{opt} option" do
|
159
169
|
name = 'created_at'
|
160
170
|
parse_opt([opt, name]) {
|
161
|
-
reader.opts.
|
171
|
+
expect(reader.opts).to include(:time_column => name)
|
162
172
|
}
|
163
173
|
end
|
164
174
|
}
|
@@ -169,7 +179,7 @@ describe FileReader do
|
|
169
179
|
it "#{opt} option" do
|
170
180
|
format = '%Y'
|
171
181
|
parse_opt([opt, format]) {
|
172
|
-
reader.opts.
|
182
|
+
expect(reader.opts).to include(:time_format => format)
|
173
183
|
}
|
174
184
|
end
|
175
185
|
}
|
@@ -180,7 +190,7 @@ describe FileReader do
|
|
180
190
|
it "--time-value option with #{value_type}" do
|
181
191
|
time = Time.now
|
182
192
|
parse_opt(['--time-value', converter.call(time)]) {
|
183
|
-
reader.opts.
|
193
|
+
expect(reader.opts).to include(:time_value => time.to_i)
|
184
194
|
}
|
185
195
|
end
|
186
196
|
}
|
@@ -191,7 +201,7 @@ describe FileReader do
|
|
191
201
|
it "#{opt} option" do
|
192
202
|
enc = 'utf-8'
|
193
203
|
parse_opt([opt, enc]) {
|
194
|
-
reader.opts.
|
204
|
+
expect(reader.opts).to include(:encoding => enc)
|
195
205
|
}
|
196
206
|
end
|
197
207
|
}
|
@@ -202,7 +212,7 @@ describe FileReader do
|
|
202
212
|
it "#{opt} option" do
|
203
213
|
format = 'gzip'
|
204
214
|
parse_opt([opt, format]) {
|
205
|
-
reader.opts.
|
215
|
+
expect(reader.opts).to include(:compress => format)
|
206
216
|
}
|
207
217
|
end
|
208
218
|
}
|
@@ -212,7 +222,7 @@ describe FileReader do
|
|
212
222
|
describe 'compose_factory' do
|
213
223
|
it 'returns Proc object' do
|
214
224
|
factory = reader.compose_factory
|
215
|
-
factory.
|
225
|
+
expect(factory).to be_an_instance_of(Proc)
|
216
226
|
end
|
217
227
|
|
218
228
|
# other specs in parse spec
|
@@ -254,7 +264,7 @@ describe FileReader do
|
|
254
264
|
parse_opt(%W(-f #{format} --time-value #{@time}) + (args || [])) {
|
255
265
|
i = 0
|
256
266
|
reader.parse(io, error) { |record|
|
257
|
-
record.
|
267
|
+
expect(record).to eq(dataset[i].merge('time' => @time))
|
258
268
|
i += 1
|
259
269
|
}
|
260
270
|
}
|
@@ -266,7 +276,7 @@ describe FileReader do
|
|
266
276
|
reader.parse(io, error) { |record|
|
267
277
|
time = record[time_column]
|
268
278
|
time = Time.parse(time).to_i if time.is_a?(String)
|
269
|
-
record.
|
279
|
+
expect(record).to eq(dataset[i].merge('time' => time))
|
270
280
|
i += 1
|
271
281
|
}
|
272
282
|
}
|
@@ -43,7 +43,7 @@ module TreasureData::Command
|
|
43
43
|
let(:out_file) { Tempfile.new('out.yml').tap{|f| f.close } }
|
44
44
|
|
45
45
|
before do
|
46
|
-
command.
|
46
|
+
allow(command).to receive(:get_client).and_return(client)
|
47
47
|
end
|
48
48
|
|
49
49
|
let(:config) {
|
@@ -58,11 +58,11 @@ module TreasureData::Command
|
|
58
58
|
include_context 'quiet_out'
|
59
59
|
|
60
60
|
before do
|
61
|
-
command.
|
61
|
+
allow(command).to receive(:prepare_bulkload_job_config).and_return(config)
|
62
62
|
end
|
63
63
|
|
64
64
|
it 'guess_plugins passed td-client' do
|
65
|
-
client.
|
65
|
+
expect(client).to receive(:bulk_load_guess).with({config: expect_config}).and_return({})
|
66
66
|
|
67
67
|
command.connector_guess(option)
|
68
68
|
end
|
@@ -82,7 +82,7 @@ module TreasureData::Command
|
|
82
82
|
}
|
83
83
|
|
84
84
|
before do
|
85
|
-
command.
|
85
|
+
allow(command).to receive(:get_client).and_return(client)
|
86
86
|
command.connector_guess(option)
|
87
87
|
end
|
88
88
|
|
@@ -118,7 +118,7 @@ module TreasureData::Command
|
|
118
118
|
|
119
119
|
before do
|
120
120
|
client = double(:client, bulk_load_preview: preview_result)
|
121
|
-
command.
|
121
|
+
allow(command).to receive(:get_client).and_return(client)
|
122
122
|
end
|
123
123
|
|
124
124
|
it 'should include too_long_column_name without truncated' do
|
@@ -141,8 +141,8 @@ module TreasureData::Command
|
|
141
141
|
|
142
142
|
before do
|
143
143
|
client = double(:client, bulk_load_issue: 1234)
|
144
|
-
command.
|
145
|
-
command.
|
144
|
+
allow(command).to receive(:get_client).and_return(client)
|
145
|
+
allow(command).to receive(:create_database_and_table_if_not_exist)
|
146
146
|
end
|
147
147
|
|
148
148
|
it 'should include too_long_column_name without truncated' do
|
@@ -154,8 +154,8 @@ module TreasureData::Command
|
|
154
154
|
let(:client) { double(:client, bulk_load_issue: 1234) }
|
155
155
|
|
156
156
|
before do
|
157
|
-
command.
|
158
|
-
client.
|
157
|
+
allow(command).to receive(:get_client).and_return(client)
|
158
|
+
allow(client).to receive(:database)
|
159
159
|
end
|
160
160
|
|
161
161
|
context 'set auto crate table option' do
|
@@ -164,7 +164,7 @@ module TreasureData::Command
|
|
164
164
|
}
|
165
165
|
|
166
166
|
it 'call create_database_and_table_if_not_exist' do
|
167
|
-
command.
|
167
|
+
expect(command).to receive(:create_database_and_table_if_not_exist)
|
168
168
|
|
169
169
|
subject
|
170
170
|
end
|
@@ -176,7 +176,7 @@ module TreasureData::Command
|
|
176
176
|
}
|
177
177
|
|
178
178
|
it 'call create_database_and_table_if_not_exist' do
|
179
|
-
command.
|
179
|
+
expect(command).not_to receive(:create_database_and_table_if_not_exist)
|
180
180
|
|
181
181
|
subject
|
182
182
|
end
|
@@ -191,8 +191,8 @@ module TreasureData::Command
|
|
191
191
|
let(:job_name) { 'job_1' }
|
192
192
|
|
193
193
|
before do
|
194
|
-
command.
|
195
|
-
client.
|
194
|
+
allow(command).to receive(:get_client).and_return(client)
|
195
|
+
allow(client).to receive(:database)
|
196
196
|
end
|
197
197
|
|
198
198
|
context 'with scheduled_time' do
|
@@ -202,7 +202,7 @@ module TreasureData::Command
|
|
202
202
|
}
|
203
203
|
|
204
204
|
it 'client call with unix time' do
|
205
|
-
client.
|
205
|
+
expect(client).to receive(:bulk_load_run).with(job_name, scheduled_time.to_i).and_return(123)
|
206
206
|
|
207
207
|
command.connector_run(option)
|
208
208
|
end
|
@@ -215,8 +215,8 @@ module TreasureData::Command
|
|
215
215
|
let(:current_time) { Time.now }
|
216
216
|
|
217
217
|
it 'client call with unix time' do
|
218
|
-
client.
|
219
|
-
command.
|
218
|
+
expect(client).to receive(:bulk_load_run).with(job_name, current_time.to_i).and_return(123)
|
219
|
+
allow(command).to receive(:current_time).and_return(current_time.to_i)
|
220
220
|
|
221
221
|
command.connector_run(option)
|
222
222
|
end
|
@@ -235,8 +235,8 @@ module TreasureData::Command
|
|
235
235
|
|
236
236
|
before do
|
237
237
|
client = double(:client)
|
238
|
-
client.
|
239
|
-
command.
|
238
|
+
allow(client).to receive(:bulk_load_history).with(name).and_return(history)
|
239
|
+
allow(command).to receive(:get_client).and_return(client)
|
240
240
|
end
|
241
241
|
|
242
242
|
context 'history is empty' do
|
@@ -308,7 +308,7 @@ module TreasureData::Command
|
|
308
308
|
}
|
309
309
|
|
310
310
|
before do
|
311
|
-
command.
|
311
|
+
allow(command).to receive(:get_client).and_return(client)
|
312
312
|
command.connector_list(option)
|
313
313
|
end
|
314
314
|
|
@@ -341,8 +341,8 @@ module TreasureData::Command
|
|
341
341
|
}
|
342
342
|
|
343
343
|
before do
|
344
|
-
command.
|
345
|
-
command.
|
344
|
+
allow(command).to receive(:get_table)
|
345
|
+
allow(command).to receive(:get_client).and_return(client)
|
346
346
|
command.connector_create(option)
|
347
347
|
end
|
348
348
|
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'td/command/common'
|
3
|
+
require 'td/command/list'
|
4
|
+
require 'td/command/export'
|
5
|
+
|
6
|
+
module TreasureData::Command
|
7
|
+
describe 'export commands' do
|
8
|
+
let(:command) {
|
9
|
+
Class.new { include TreasureData::Command }.new
|
10
|
+
}
|
11
|
+
let(:stdout_io) { StringIO.new }
|
12
|
+
let(:stderr_io) { StringIO.new }
|
13
|
+
|
14
|
+
around do |example|
|
15
|
+
stdout = $stdout.dup
|
16
|
+
stderr = $stderr.dup
|
17
|
+
|
18
|
+
begin
|
19
|
+
$stdout = stdout_io
|
20
|
+
$stderr = stderr_io
|
21
|
+
|
22
|
+
example.run
|
23
|
+
ensure
|
24
|
+
$stdout = stdout
|
25
|
+
$stderr = stderr
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '#table_export' do
|
30
|
+
let(:option) {
|
31
|
+
List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, option_list, true)
|
32
|
+
}
|
33
|
+
let(:option_with_encryption) {
|
34
|
+
ops = option_list
|
35
|
+
ops.push "-e"
|
36
|
+
ops.push encryption
|
37
|
+
List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, ops, true)
|
38
|
+
}
|
39
|
+
let(:option_with_wrong_encryption) {
|
40
|
+
ops = option_list
|
41
|
+
ops.push "-e"
|
42
|
+
ops.push wrong_encryption
|
43
|
+
List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, ops, true)
|
44
|
+
}
|
45
|
+
let(:option_list) { [database, table, "-b", bucket, "-p", path, "-k", key, "-s", pass, "-F", format] }
|
46
|
+
let(:database) { 'database' }
|
47
|
+
let(:table) { 'table' }
|
48
|
+
let(:bucket) { 'bucket' }
|
49
|
+
let(:path) { 'path' }
|
50
|
+
let(:key) { 'key' }
|
51
|
+
let(:pass) { 'pass' }
|
52
|
+
let(:format) { 'tsv.gz' }
|
53
|
+
let(:encryption) { 's3' }
|
54
|
+
let(:wrong_encryption) { 's3s3' }
|
55
|
+
let(:job_id) { 111 }
|
56
|
+
|
57
|
+
before do
|
58
|
+
client = double(:client)
|
59
|
+
job = double(:job, job_id: job_id)
|
60
|
+
allow(client).to receive(:export).and_return(job)
|
61
|
+
table = double(:table)
|
62
|
+
|
63
|
+
allow(command).to receive(:get_client).and_return(client)
|
64
|
+
allow(command).to receive(:get_table).and_return(table)
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'export table without encryption' do
|
68
|
+
expect {
|
69
|
+
command.table_export(option)
|
70
|
+
}.to_not raise_exception
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'export table with encryption' do
|
74
|
+
expect {
|
75
|
+
command.table_export(option_with_encryption)
|
76
|
+
}.to_not raise_exception
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'fail to export table with wrong encryption' do
|
80
|
+
expect {
|
81
|
+
command.table_export(option_with_wrong_encryption)
|
82
|
+
}.to raise_exception
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|