td 0.10.83 → 0.10.84
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/ChangeLog +8 -0
- data/lib/td/command/bulk_import.rb +17 -2
- data/lib/td/command/common.rb +8 -0
- data/lib/td/command/export.rb +4 -2
- data/lib/td/command/import.rb +9 -2
- data/lib/td/command/job.rb +2 -2
- data/lib/td/version.rb +1 -1
- data/td.gemspec +1 -1
- metadata +21 -49
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c7c60a874b7228266a8436132aae4d0e2438b3e5
|
4
|
+
data.tar.gz: 9caacd3a0d3065d8fd4dca7bc7bae1cd29b9bbd5
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: baa729bcbfb73760f2c93cc70e6fefccc69b1cae9a41c62e7c869372bf4124975649e0abffb5db15b79156bfb59e59ab52875634bf179da86fab129979c7cdab
|
7
|
+
data.tar.gz: 10328c283f1b8a9a3e4d2b9ad23187da08b0b0391ae237fdd5fbe8ad4c610784f0d8a2f2ca8d00c758f5afac757426965545e5811b406df943769a826b9a27d7
|
data/ChangeLog
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
== 2013-07-25 version 0.10.84
|
2
|
+
|
3
|
+
* bulk_import now checks first argument is correct session name
|
4
|
+
* Convert bignum into string type. Please use cast in your query.
|
5
|
+
* Improve error message of export -F option
|
6
|
+
* td-client v0.8.53
|
7
|
+
|
8
|
+
|
1
9
|
== 2013-07-03 version 0.10.83
|
2
10
|
|
3
11
|
* Add -x option to query, exlude result from automatically being downloaded
|
@@ -129,6 +129,13 @@ module Command
|
|
129
129
|
|
130
130
|
name, *files = op.cmd_parse
|
131
131
|
|
132
|
+
# validate the session
|
133
|
+
bi = get_client.bulk_imports.find {|bi| name == bi.name }
|
134
|
+
unless bi
|
135
|
+
$stderr.puts "Bulk import session '#{name}' does not exist. Please check the first argument of the command."
|
136
|
+
exit 1
|
137
|
+
end
|
138
|
+
|
132
139
|
parallel = 1 if parallel <= 1
|
133
140
|
parallel = 8 if parallel >= 8
|
134
141
|
|
@@ -320,7 +327,6 @@ module Command
|
|
320
327
|
$stderr.puts "Bulk import session '#{name}' is unfrozen."
|
321
328
|
end
|
322
329
|
|
323
|
-
|
324
330
|
PART_SPLIT_SIZE = 16*1024*1024
|
325
331
|
|
326
332
|
def bulk_import_prepare_parts(op)
|
@@ -364,6 +370,7 @@ module Command
|
|
364
370
|
$stderr.puts "#{reason}"
|
365
371
|
end
|
366
372
|
}
|
373
|
+
has_bignum = false
|
367
374
|
|
368
375
|
# TODO multi process
|
369
376
|
files.each {|ifname|
|
@@ -387,7 +394,13 @@ module Command
|
|
387
394
|
$stderr.puts " sample: #{Time.at(t).utc} #{Yajl.dump(record)}"
|
388
395
|
end
|
389
396
|
|
390
|
-
|
397
|
+
entry = begin
|
398
|
+
record.to_msgpack
|
399
|
+
rescue RangeError
|
400
|
+
has_bignum = true
|
401
|
+
TreasureData::API.normalized_msgpack(record)
|
402
|
+
end
|
403
|
+
zout.write(entry)
|
391
404
|
record_num += 1
|
392
405
|
|
393
406
|
if out.size > split_size
|
@@ -406,6 +419,8 @@ module Command
|
|
406
419
|
$stderr.puts " #{ifname}: #{record_num} entries."
|
407
420
|
}
|
408
421
|
}
|
422
|
+
|
423
|
+
$stderr.puts normalized_message if has_bignum
|
409
424
|
end
|
410
425
|
|
411
426
|
def bulk_import_upload_parts2(op)
|
data/lib/td/command/common.rb
CHANGED
@@ -46,6 +46,14 @@ module Command
|
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
+
def normalized_message
|
50
|
+
<<EOS
|
51
|
+
Your event has large number larger than 2^64.
|
52
|
+
These numbers are converted into string type.
|
53
|
+
So you should use cast operator, e.g. cast(v['key'] as decimal), in your query.
|
54
|
+
EOS
|
55
|
+
end
|
56
|
+
|
49
57
|
#def cmd_render_tree(nodes, *opts)
|
50
58
|
# require 'hirb'
|
51
59
|
# Hirb::Helpers::Tree.render(nodes, *opts)
|
data/lib/td/command/export.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
|
2
2
|
module TreasureData
|
3
3
|
module Command
|
4
|
+
SUPPORTED_FORMATS = %W[json.gz line-json.gz]
|
4
5
|
|
5
6
|
def table_export(op)
|
6
7
|
org = nil
|
@@ -9,7 +10,7 @@ module Command
|
|
9
10
|
s3_bucket = nil
|
10
11
|
aws_access_key_id = nil
|
11
12
|
aws_secret_access_key = nil
|
12
|
-
file_format =
|
13
|
+
file_format = nil
|
13
14
|
|
14
15
|
op.on('-g', '--org ORGANIZATION', "export the data under this organization") {|s|
|
15
16
|
org = s
|
@@ -30,7 +31,8 @@ module Command
|
|
30
31
|
aws_secret_access_key = s
|
31
32
|
}
|
32
33
|
op.on('-F', '--file-format FILE_FORMAT', 'file format for exported data, either json.gz (default) or line-json.gz') { |s|
|
33
|
-
|
34
|
+
raise ArgumentError, "#{s} is not a supported file format" unless SUPPORTED_FORMATS.include?(s)
|
35
|
+
file_format = s
|
34
36
|
}
|
35
37
|
|
36
38
|
db_name, table_name = op.cmd_parse
|
data/lib/td/command/import.rb
CHANGED
@@ -136,8 +136,15 @@ module Command
|
|
136
136
|
|
137
137
|
n = 0
|
138
138
|
x = 0
|
139
|
+
has_bignum = false
|
139
140
|
parser.call(file, path) {|record|
|
140
|
-
|
141
|
+
entry = begin
|
142
|
+
record.to_msgpack
|
143
|
+
rescue RangeError
|
144
|
+
has_bignum = true
|
145
|
+
TreasureData::API.normalized_msgpack(record)
|
146
|
+
end
|
147
|
+
writer.write entry
|
141
148
|
|
142
149
|
n += 1
|
143
150
|
x += 1
|
@@ -176,7 +183,7 @@ module Command
|
|
176
183
|
end
|
177
184
|
|
178
185
|
puts " imported #{n} entries from #{path}."
|
179
|
-
|
186
|
+
$stderr.puts normalized_message if has_bignum
|
180
187
|
ensure
|
181
188
|
out.close rescue nil
|
182
189
|
writer.close rescue nil
|
data/lib/td/command/job.rb
CHANGED
@@ -130,13 +130,13 @@ module Command
|
|
130
130
|
|
131
131
|
if wait && !job.finished?
|
132
132
|
wait_job(job)
|
133
|
-
if job.success? && [:hive, :pig].include?(job.type) && !exclude
|
133
|
+
if job.success? && [:hive, :pig, :impala].include?(job.type) && !exclude
|
134
134
|
puts "Result :"
|
135
135
|
show_result(job, output, format, render_opts)
|
136
136
|
end
|
137
137
|
|
138
138
|
else
|
139
|
-
if job.success? && [:hive, :pig].include?(job.type) && !exclude
|
139
|
+
if job.success? && [:hive, :pig, :impala].include?(job.type) && !exclude
|
140
140
|
puts "Result :"
|
141
141
|
show_result(job, output, format, render_opts)
|
142
142
|
end
|
data/lib/td/version.rb
CHANGED
data/td.gemspec
CHANGED
@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.add_dependency "yajl-ruby", "~> 1.1.0"
|
21
21
|
gem.add_dependency "hirb", ">= 0.4.5"
|
22
22
|
gem.add_dependency "parallel", "~> 0.6.1"
|
23
|
-
gem.add_dependency "td-client", "~> 0.8.
|
23
|
+
gem.add_dependency "td-client", "~> 0.8.53"
|
24
24
|
gem.add_dependency "td-logger", "~> 0.3.21"
|
25
25
|
gem.add_dependency "rubyzip", "~> 0.9.9"
|
26
26
|
gem.add_development_dependency "rake", "~> 0.9"
|
metadata
CHANGED
@@ -1,34 +1,32 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
5
|
-
prerelease:
|
4
|
+
version: 0.10.84
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Treasure Data, Inc.
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2013-07-
|
11
|
+
date: 2013-07-25 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: msgpack
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - '>='
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: 0.4.4
|
22
|
-
- -
|
20
|
+
- - '!='
|
23
21
|
- !ruby/object:Gem::Version
|
24
22
|
version: 0.5.0
|
25
|
-
- -
|
23
|
+
- - '!='
|
26
24
|
- !ruby/object:Gem::Version
|
27
25
|
version: 0.5.1
|
28
|
-
- -
|
26
|
+
- - '!='
|
29
27
|
- !ruby/object:Gem::Version
|
30
28
|
version: 0.5.2
|
31
|
-
- -
|
29
|
+
- - '!='
|
32
30
|
- !ruby/object:Gem::Version
|
33
31
|
version: 0.5.3
|
34
32
|
- - <
|
@@ -37,21 +35,20 @@ dependencies:
|
|
37
35
|
type: :runtime
|
38
36
|
prerelease: false
|
39
37
|
version_requirements: !ruby/object:Gem::Requirement
|
40
|
-
none: false
|
41
38
|
requirements:
|
42
|
-
- -
|
39
|
+
- - '>='
|
43
40
|
- !ruby/object:Gem::Version
|
44
41
|
version: 0.4.4
|
45
|
-
- -
|
42
|
+
- - '!='
|
46
43
|
- !ruby/object:Gem::Version
|
47
44
|
version: 0.5.0
|
48
|
-
- -
|
45
|
+
- - '!='
|
49
46
|
- !ruby/object:Gem::Version
|
50
47
|
version: 0.5.1
|
51
|
-
- -
|
48
|
+
- - '!='
|
52
49
|
- !ruby/object:Gem::Version
|
53
50
|
version: 0.5.2
|
54
|
-
- -
|
51
|
+
- - '!='
|
55
52
|
- !ruby/object:Gem::Version
|
56
53
|
version: 0.5.3
|
57
54
|
- - <
|
@@ -60,7 +57,6 @@ dependencies:
|
|
60
57
|
- !ruby/object:Gem::Dependency
|
61
58
|
name: yajl-ruby
|
62
59
|
requirement: !ruby/object:Gem::Requirement
|
63
|
-
none: false
|
64
60
|
requirements:
|
65
61
|
- - ~>
|
66
62
|
- !ruby/object:Gem::Version
|
@@ -68,7 +64,6 @@ dependencies:
|
|
68
64
|
type: :runtime
|
69
65
|
prerelease: false
|
70
66
|
version_requirements: !ruby/object:Gem::Requirement
|
71
|
-
none: false
|
72
67
|
requirements:
|
73
68
|
- - ~>
|
74
69
|
- !ruby/object:Gem::Version
|
@@ -76,23 +71,20 @@ dependencies:
|
|
76
71
|
- !ruby/object:Gem::Dependency
|
77
72
|
name: hirb
|
78
73
|
requirement: !ruby/object:Gem::Requirement
|
79
|
-
none: false
|
80
74
|
requirements:
|
81
|
-
- -
|
75
|
+
- - '>='
|
82
76
|
- !ruby/object:Gem::Version
|
83
77
|
version: 0.4.5
|
84
78
|
type: :runtime
|
85
79
|
prerelease: false
|
86
80
|
version_requirements: !ruby/object:Gem::Requirement
|
87
|
-
none: false
|
88
81
|
requirements:
|
89
|
-
- -
|
82
|
+
- - '>='
|
90
83
|
- !ruby/object:Gem::Version
|
91
84
|
version: 0.4.5
|
92
85
|
- !ruby/object:Gem::Dependency
|
93
86
|
name: parallel
|
94
87
|
requirement: !ruby/object:Gem::Requirement
|
95
|
-
none: false
|
96
88
|
requirements:
|
97
89
|
- - ~>
|
98
90
|
- !ruby/object:Gem::Version
|
@@ -100,7 +92,6 @@ dependencies:
|
|
100
92
|
type: :runtime
|
101
93
|
prerelease: false
|
102
94
|
version_requirements: !ruby/object:Gem::Requirement
|
103
|
-
none: false
|
104
95
|
requirements:
|
105
96
|
- - ~>
|
106
97
|
- !ruby/object:Gem::Version
|
@@ -108,23 +99,20 @@ dependencies:
|
|
108
99
|
- !ruby/object:Gem::Dependency
|
109
100
|
name: td-client
|
110
101
|
requirement: !ruby/object:Gem::Requirement
|
111
|
-
none: false
|
112
102
|
requirements:
|
113
103
|
- - ~>
|
114
104
|
- !ruby/object:Gem::Version
|
115
|
-
version: 0.8.
|
105
|
+
version: 0.8.53
|
116
106
|
type: :runtime
|
117
107
|
prerelease: false
|
118
108
|
version_requirements: !ruby/object:Gem::Requirement
|
119
|
-
none: false
|
120
109
|
requirements:
|
121
110
|
- - ~>
|
122
111
|
- !ruby/object:Gem::Version
|
123
|
-
version: 0.8.
|
112
|
+
version: 0.8.53
|
124
113
|
- !ruby/object:Gem::Dependency
|
125
114
|
name: td-logger
|
126
115
|
requirement: !ruby/object:Gem::Requirement
|
127
|
-
none: false
|
128
116
|
requirements:
|
129
117
|
- - ~>
|
130
118
|
- !ruby/object:Gem::Version
|
@@ -132,7 +120,6 @@ dependencies:
|
|
132
120
|
type: :runtime
|
133
121
|
prerelease: false
|
134
122
|
version_requirements: !ruby/object:Gem::Requirement
|
135
|
-
none: false
|
136
123
|
requirements:
|
137
124
|
- - ~>
|
138
125
|
- !ruby/object:Gem::Version
|
@@ -140,7 +127,6 @@ dependencies:
|
|
140
127
|
- !ruby/object:Gem::Dependency
|
141
128
|
name: rubyzip
|
142
129
|
requirement: !ruby/object:Gem::Requirement
|
143
|
-
none: false
|
144
130
|
requirements:
|
145
131
|
- - ~>
|
146
132
|
- !ruby/object:Gem::Version
|
@@ -148,7 +134,6 @@ dependencies:
|
|
148
134
|
type: :runtime
|
149
135
|
prerelease: false
|
150
136
|
version_requirements: !ruby/object:Gem::Requirement
|
151
|
-
none: false
|
152
137
|
requirements:
|
153
138
|
- - ~>
|
154
139
|
- !ruby/object:Gem::Version
|
@@ -156,7 +141,6 @@ dependencies:
|
|
156
141
|
- !ruby/object:Gem::Dependency
|
157
142
|
name: rake
|
158
143
|
requirement: !ruby/object:Gem::Requirement
|
159
|
-
none: false
|
160
144
|
requirements:
|
161
145
|
- - ~>
|
162
146
|
- !ruby/object:Gem::Version
|
@@ -164,7 +148,6 @@ dependencies:
|
|
164
148
|
type: :development
|
165
149
|
prerelease: false
|
166
150
|
version_requirements: !ruby/object:Gem::Requirement
|
167
|
-
none: false
|
168
151
|
requirements:
|
169
152
|
- - ~>
|
170
153
|
- !ruby/object:Gem::Version
|
@@ -172,7 +155,6 @@ dependencies:
|
|
172
155
|
- !ruby/object:Gem::Dependency
|
173
156
|
name: rspec
|
174
157
|
requirement: !ruby/object:Gem::Requirement
|
175
|
-
none: false
|
176
158
|
requirements:
|
177
159
|
- - ~>
|
178
160
|
- !ruby/object:Gem::Version
|
@@ -180,7 +162,6 @@ dependencies:
|
|
180
162
|
type: :development
|
181
163
|
prerelease: false
|
182
164
|
version_requirements: !ruby/object:Gem::Requirement
|
183
|
-
none: false
|
184
165
|
requirements:
|
185
166
|
- - ~>
|
186
167
|
- !ruby/object:Gem::Version
|
@@ -188,7 +169,6 @@ dependencies:
|
|
188
169
|
- !ruby/object:Gem::Dependency
|
189
170
|
name: simplecov
|
190
171
|
requirement: !ruby/object:Gem::Requirement
|
191
|
-
none: false
|
192
172
|
requirements:
|
193
173
|
- - ~>
|
194
174
|
- !ruby/object:Gem::Version
|
@@ -196,7 +176,6 @@ dependencies:
|
|
196
176
|
type: :development
|
197
177
|
prerelease: false
|
198
178
|
version_requirements: !ruby/object:Gem::Requirement
|
199
|
-
none: false
|
200
179
|
requirements:
|
201
180
|
- - ~>
|
202
181
|
- !ruby/object:Gem::Version
|
@@ -280,33 +259,26 @@ files:
|
|
280
259
|
- td.gemspec
|
281
260
|
homepage: http://treasure-data.com/
|
282
261
|
licenses: []
|
262
|
+
metadata: {}
|
283
263
|
post_install_message:
|
284
264
|
rdoc_options: []
|
285
265
|
require_paths:
|
286
266
|
- lib
|
287
267
|
required_ruby_version: !ruby/object:Gem::Requirement
|
288
|
-
none: false
|
289
268
|
requirements:
|
290
|
-
- -
|
269
|
+
- - '>='
|
291
270
|
- !ruby/object:Gem::Version
|
292
271
|
version: '0'
|
293
|
-
segments:
|
294
|
-
- 0
|
295
|
-
hash: 2192841065955335729
|
296
272
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
297
|
-
none: false
|
298
273
|
requirements:
|
299
|
-
- -
|
274
|
+
- - '>='
|
300
275
|
- !ruby/object:Gem::Version
|
301
276
|
version: '0'
|
302
|
-
segments:
|
303
|
-
- 0
|
304
|
-
hash: 2192841065955335729
|
305
277
|
requirements: []
|
306
278
|
rubyforge_project:
|
307
|
-
rubygems_version:
|
279
|
+
rubygems_version: 2.0.2
|
308
280
|
signing_key:
|
309
|
-
specification_version:
|
281
|
+
specification_version: 4
|
310
282
|
summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
|
311
283
|
test_files:
|
312
284
|
- spec/file_reader/filter_spec.rb
|