td 0.10.23 → 0.10.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +16 -0
- data/lib/td/command/bulk_import.rb +205 -0
- data/lib/td/command/list.rb +15 -0
- data/lib/td/command/table.rb +2 -2
- data/lib/td/version.rb +1 -1
- metadata +14 -13
data/ChangeLog
CHANGED
|
@@ -1,4 +1,20 @@
|
|
|
1
1
|
|
|
2
|
+
== 2012-05-10 version 0.10.24
|
|
3
|
+
|
|
4
|
+
* Added bulk_import subcommands
|
|
5
|
+
* bulk_import:list
|
|
6
|
+
* bulk_import:show <name>
|
|
7
|
+
* bulk_import:create <name> <db> <table>
|
|
8
|
+
* bulk_import:upload_part <name> <id> <path.msgpack.gz>
|
|
9
|
+
* bulk_import:delete_part <name> <id>
|
|
10
|
+
* bulk_import:perform <name>
|
|
11
|
+
* bulk_import:error_records <name>
|
|
12
|
+
* bulk_import:commit <name>
|
|
13
|
+
* bulk_import:delete <name>
|
|
14
|
+
* bulk_import:freeze <name>
|
|
15
|
+
* bulk_import:unfreeze <name>
|
|
16
|
+
|
|
17
|
+
|
|
2
18
|
== 2012-04-26 version 0.10.23
|
|
3
19
|
|
|
4
20
|
* Added result:list, result:create and result:delete subcommands
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
|
|
2
|
+
module TreasureData
|
|
3
|
+
module Command
|
|
4
|
+
|
|
5
|
+
def bulk_import_list(op)
|
|
6
|
+
op.cmd_parse
|
|
7
|
+
|
|
8
|
+
client = get_client
|
|
9
|
+
|
|
10
|
+
bis = client.bulk_imports
|
|
11
|
+
|
|
12
|
+
rows = []
|
|
13
|
+
bis.each {|bi|
|
|
14
|
+
rows << {:Name=>bi.name, :Table=>"#{bi.database}.#{bi.table}", :Status=>bi.status.to_s.capitalize, :Frozen=>bi.upload_frozen? ? 'frozen' : '', :JobID=>bi.job_id, :"Valid Parts"=>bi.valid_parts, :"Error Parts"=>bi.error_parts, :"Valid Records"=>bi.valid_records, :"Error Records"=>bi.error_records}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
puts cmd_render_table(rows, :fields => [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"])
|
|
18
|
+
|
|
19
|
+
if rows.empty?
|
|
20
|
+
$stderr.puts "There are no bulk import sessions."
|
|
21
|
+
$stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def bulk_import_create(op)
|
|
26
|
+
name, db_name, table_name = op.cmd_parse
|
|
27
|
+
|
|
28
|
+
client = get_client
|
|
29
|
+
|
|
30
|
+
table = get_table(client, db_name, table_name)
|
|
31
|
+
|
|
32
|
+
client.create_bulk_import(name, db_name, table_name)
|
|
33
|
+
|
|
34
|
+
$stderr.puts "Bulk import session '#{name}' is created."
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def bulk_import_delete(op)
|
|
38
|
+
name = op.cmd_parse
|
|
39
|
+
|
|
40
|
+
client = get_client
|
|
41
|
+
|
|
42
|
+
begin
|
|
43
|
+
client.delete_bulk_import(name)
|
|
44
|
+
rescue NotFoundError
|
|
45
|
+
cmd_debug_error $!
|
|
46
|
+
$stderr.puts "Bulk import session '#{name}' does not exist."
|
|
47
|
+
exit 1
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
$stderr.puts "Bulk import session '#{name}' is deleted."
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def bulk_import_show(op)
|
|
54
|
+
name = op.cmd_parse
|
|
55
|
+
|
|
56
|
+
client = get_client
|
|
57
|
+
|
|
58
|
+
bis = client.bulk_imports
|
|
59
|
+
bi = bis.find {|bi| name == bi.name }
|
|
60
|
+
unless bi
|
|
61
|
+
$stderr.puts "Bulk import session '#{name}' does not exist."
|
|
62
|
+
$stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
|
|
63
|
+
exit 1
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
$stderr.puts "Name : #{bi.name}"
|
|
67
|
+
$stderr.puts "Database : #{bi.database}"
|
|
68
|
+
$stderr.puts "Table : #{bi.table}"
|
|
69
|
+
$stderr.puts "Status : #{bi.status.to_s.capitalize}"
|
|
70
|
+
$stderr.puts "Frozen : #{bi.upload_frozen?}"
|
|
71
|
+
$stderr.puts "JobID : #{bi.job_id}"
|
|
72
|
+
$stderr.puts "Valid Records: #{bi.valid_records}"
|
|
73
|
+
$stderr.puts "Error Records: #{bi.error_records}"
|
|
74
|
+
$stderr.puts "Valid Parts : #{bi.valid_parts}"
|
|
75
|
+
$stderr.puts "Error Parts : #{bi.error_parts}"
|
|
76
|
+
$stderr.puts "Uploaded Parts :"
|
|
77
|
+
|
|
78
|
+
list = client.list_bulk_import_parts(name)
|
|
79
|
+
list.each {|name|
|
|
80
|
+
puts " #{name}"
|
|
81
|
+
}
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def bulk_import_upload_part(op)
|
|
85
|
+
name, part_name, path = op.cmd_parse
|
|
86
|
+
|
|
87
|
+
client = get_client
|
|
88
|
+
|
|
89
|
+
File.open(path) {|is|
|
|
90
|
+
client.bulk_import_upload_part(name, part_name, is, is.size)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
$stderr.puts "Part '#{part_name}' is uploaded."
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def bulk_import_delete_part(op)
|
|
97
|
+
name, part_name = op.cmd_parse
|
|
98
|
+
|
|
99
|
+
client = get_client
|
|
100
|
+
|
|
101
|
+
client.bulk_import_delete_part(name, part_name)
|
|
102
|
+
|
|
103
|
+
$stderr.puts "Part '#{part_name}' is deleted."
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def bulk_import_perform(op)
|
|
107
|
+
wait = false
|
|
108
|
+
force = false
|
|
109
|
+
|
|
110
|
+
op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
|
|
111
|
+
wait = b
|
|
112
|
+
}
|
|
113
|
+
op.on('-f', '--force', 'force start performing', TrueClass) {|b|
|
|
114
|
+
force = b
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
name = op.cmd_parse
|
|
118
|
+
|
|
119
|
+
client = get_client
|
|
120
|
+
|
|
121
|
+
unless force
|
|
122
|
+
bis = client.bulk_imports
|
|
123
|
+
bi = bis.find {|bi| name == bi.name }
|
|
124
|
+
if bi
|
|
125
|
+
if bi.status == 'performing'
|
|
126
|
+
$stderr.puts "Bulk import session '#{name}' is already performing."
|
|
127
|
+
$stderr.puts "Add '-f' option to force start."
|
|
128
|
+
$stderr.puts "Use '#{$prog} job:kill #{bi.job_id}' to cancel the last trial."
|
|
129
|
+
exit 1
|
|
130
|
+
elsif bi.status == 'ready'
|
|
131
|
+
$stderr.puts "Bulk import session '#{name}' is already ready to commit."
|
|
132
|
+
$stderr.puts "Add '-f' option to force start."
|
|
133
|
+
exit 1
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
job = client.perform_bulk_import(name)
|
|
139
|
+
|
|
140
|
+
$stderr.puts "Job #{job.job_id} is queued."
|
|
141
|
+
$stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
|
|
142
|
+
|
|
143
|
+
if wait
|
|
144
|
+
require 'td/command/job' # wait_job
|
|
145
|
+
wait_job(job)
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def bulk_import_commit(op)
|
|
150
|
+
name = op.cmd_parse
|
|
151
|
+
|
|
152
|
+
client = get_client
|
|
153
|
+
|
|
154
|
+
job = client.commit_bulk_import(name)
|
|
155
|
+
|
|
156
|
+
$stderr.puts "Bulk import session '#{name}' started to commit."
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def bulk_import_error_records(op)
|
|
160
|
+
name = op.cmd_parse
|
|
161
|
+
|
|
162
|
+
client = get_client
|
|
163
|
+
|
|
164
|
+
bis = client.bulk_imports
|
|
165
|
+
bi = bis.find {|bi| name == bi.name }
|
|
166
|
+
unless bi
|
|
167
|
+
$stderr.puts "Bulk import session '#{name}' does not exist."
|
|
168
|
+
$stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
|
|
169
|
+
exit 1
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
if bi.status == "uploading" || bi.status == "performing"
|
|
173
|
+
$stderr.puts "Bulk import session '#{name}' is not performed."
|
|
174
|
+
$stderr.puts "Use '#{$prog} bulk_import:perform <name>' to run."
|
|
175
|
+
exit 1
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
client.bulk_import_error_records(name) {|r|
|
|
179
|
+
puts r.to_json
|
|
180
|
+
}
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def bulk_import_freeze(op)
|
|
184
|
+
name = op.cmd_parse
|
|
185
|
+
|
|
186
|
+
client = get_client
|
|
187
|
+
|
|
188
|
+
client.freeze_bulk_import(name)
|
|
189
|
+
|
|
190
|
+
$stderr.puts "Bulk import session '#{name}' is frozen."
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def bulk_import_unfreeze(op)
|
|
194
|
+
name = op.cmd_parse
|
|
195
|
+
|
|
196
|
+
client = get_client
|
|
197
|
+
|
|
198
|
+
client.unfreeze_bulk_import(name)
|
|
199
|
+
|
|
200
|
+
$stderr.puts "Bulk import session '#{name}' is unfrozen."
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
data/lib/td/command/list.rb
CHANGED
|
@@ -224,6 +224,18 @@ module List
|
|
|
224
224
|
add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', 'table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY'
|
|
225
225
|
add_list 'table:tail', %w[db table], 'Get recently imported logs', 'table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30'
|
|
226
226
|
|
|
227
|
+
add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
|
|
228
|
+
add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
|
|
229
|
+
add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
|
|
230
|
+
add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
|
|
231
|
+
add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
|
|
232
|
+
add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'
|
|
233
|
+
add_list 'bulk_import:error_records', %w[name], 'Show records which did not pass validations', 'bulk_import:error_records logs_201201'
|
|
234
|
+
add_list 'bulk_import:commit', %w[name], 'Start to commit a performed bulk import session', 'bulk_import:commit logs_201201'
|
|
235
|
+
add_list 'bulk_import:delete', %w[name], 'Delete a bulk import session', 'bulk_import:delete logs_201201'
|
|
236
|
+
add_list 'bulk_import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', 'bulk_import:freeze logs_201201'
|
|
237
|
+
add_list 'bulk_import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'bulk_import:unfreeze logs_201201'
|
|
238
|
+
|
|
227
239
|
add_list 'result:list', %w[], 'Show list of result URLs', 'result:list', 'results'
|
|
228
240
|
add_list 'result:show', %w[name], 'Describe information of a result URL', 'result mydb'
|
|
229
241
|
add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create mydb mysql://my-server/mydb'
|
|
@@ -298,6 +310,9 @@ module List
|
|
|
298
310
|
add_alias 'scheds', 'sched:list'
|
|
299
311
|
add_alias 'schedules', 'sched:list'
|
|
300
312
|
|
|
313
|
+
add_alias 'bulk_import', 'bulk_import:show'
|
|
314
|
+
add_alias 'bulk_imports', 'bulk_import:list'
|
|
315
|
+
|
|
301
316
|
add_alias 'job', 'job:show'
|
|
302
317
|
add_alias 'jobs', 'job:list'
|
|
303
318
|
add_alias 'kill', 'job:kill'
|
data/lib/td/command/table.rb
CHANGED
|
@@ -98,13 +98,13 @@ module Command
|
|
|
98
98
|
if rows.empty?
|
|
99
99
|
if db_name
|
|
100
100
|
$stderr.puts "Database '#{db_name}' has no tables."
|
|
101
|
-
$stderr.puts "Use '#{$prog} table:create <db
|
|
101
|
+
$stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
|
|
102
102
|
elsif dbs.empty?
|
|
103
103
|
$stderr.puts "There are no databases."
|
|
104
104
|
$stderr.puts "Use '#{$prog} db:create <db>' to create a database."
|
|
105
105
|
else
|
|
106
106
|
$stderr.puts "There are no tables."
|
|
107
|
-
$stderr.puts "Use '#{$prog} table:create <db
|
|
107
|
+
$stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
|
|
108
108
|
end
|
|
109
109
|
end
|
|
110
110
|
end
|
data/lib/td/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: td
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.10.
|
|
4
|
+
version: 0.10.24
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,11 +9,11 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2012-
|
|
12
|
+
date: 2012-05-10 00:00:00.000000000Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: msgpack
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70362122344260 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ~>
|
|
@@ -21,10 +21,10 @@ dependencies:
|
|
|
21
21
|
version: 0.4.4
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70362122344260
|
|
25
25
|
- !ruby/object:Gem::Dependency
|
|
26
26
|
name: json
|
|
27
|
-
requirement: &
|
|
27
|
+
requirement: &70362122343780 !ruby/object:Gem::Requirement
|
|
28
28
|
none: false
|
|
29
29
|
requirements:
|
|
30
30
|
- - ! '>='
|
|
@@ -32,10 +32,10 @@ dependencies:
|
|
|
32
32
|
version: 1.4.3
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
|
-
version_requirements: *
|
|
35
|
+
version_requirements: *70362122343780
|
|
36
36
|
- !ruby/object:Gem::Dependency
|
|
37
37
|
name: hirb
|
|
38
|
-
requirement: &
|
|
38
|
+
requirement: &70362122343300 !ruby/object:Gem::Requirement
|
|
39
39
|
none: false
|
|
40
40
|
requirements:
|
|
41
41
|
- - ! '>='
|
|
@@ -43,21 +43,21 @@ dependencies:
|
|
|
43
43
|
version: 0.4.5
|
|
44
44
|
type: :runtime
|
|
45
45
|
prerelease: false
|
|
46
|
-
version_requirements: *
|
|
46
|
+
version_requirements: *70362122343300
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
name: td-client
|
|
49
|
-
requirement: &
|
|
49
|
+
requirement: &70362122342820 !ruby/object:Gem::Requirement
|
|
50
50
|
none: false
|
|
51
51
|
requirements:
|
|
52
52
|
- - ~>
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: 0.8.
|
|
54
|
+
version: 0.8.16
|
|
55
55
|
type: :runtime
|
|
56
56
|
prerelease: false
|
|
57
|
-
version_requirements: *
|
|
57
|
+
version_requirements: *70362122342820
|
|
58
58
|
- !ruby/object:Gem::Dependency
|
|
59
59
|
name: td-logger
|
|
60
|
-
requirement: &
|
|
60
|
+
requirement: &70362122342340 !ruby/object:Gem::Requirement
|
|
61
61
|
none: false
|
|
62
62
|
requirements:
|
|
63
63
|
- - ~>
|
|
@@ -65,7 +65,7 @@ dependencies:
|
|
|
65
65
|
version: 0.3.8
|
|
66
66
|
type: :runtime
|
|
67
67
|
prerelease: false
|
|
68
|
-
version_requirements: *
|
|
68
|
+
version_requirements: *70362122342340
|
|
69
69
|
description:
|
|
70
70
|
email:
|
|
71
71
|
executables:
|
|
@@ -81,6 +81,7 @@ files:
|
|
|
81
81
|
- lib/td/command/account.rb
|
|
82
82
|
- lib/td/command/aggr.rb
|
|
83
83
|
- lib/td/command/apikey.rb
|
|
84
|
+
- lib/td/command/bulk_import.rb
|
|
84
85
|
- lib/td/command/common.rb
|
|
85
86
|
- lib/td/command/db.rb
|
|
86
87
|
- lib/td/command/export.rb
|