td-client 0.8.67 → 0.8.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/td/client.rb +12 -2
- data/lib/td/client/api.rb +51 -1040
- data/lib/td/client/api/access_control.rb +62 -0
- data/lib/td/client/api/account.rb +41 -0
- data/lib/td/client/api/bulk_import.rb +154 -0
- data/lib/td/client/api/database.rb +47 -0
- data/lib/td/client/api/export.rb +21 -0
- data/lib/td/client/api/import.rb +31 -0
- data/lib/td/client/api/job.rb +251 -0
- data/lib/td/client/api/partial_delete.rb +21 -0
- data/lib/td/client/api/result.rb +41 -0
- data/lib/td/client/api/schedule.rb +106 -0
- data/lib/td/client/api/server_status.rb +20 -0
- data/lib/td/client/api/table.rb +123 -0
- data/lib/td/client/api/user.rb +117 -0
- data/lib/td/client/api_error.rb +26 -0
- data/lib/td/client/model.rb +7 -5
- data/lib/td/client/version.rb +1 -1
- data/lib/td/core_ext/openssl/ssl/sslcontext/set_params.rb +18 -0
- data/spec/spec_helper.rb +8 -4
- data/spec/td/client/access_control_api_spec.rb +37 -0
- data/spec/td/client/account_api_spec.rb +34 -0
- data/spec/td/client/api_ssl_connection_spec.rb +10 -7
- data/spec/td/client/bulk_import_spec.rb +144 -0
- data/spec/td/client/import_api_spec.rb +73 -0
- data/spec/td/client/job_api_spec.rb +124 -1
- data/spec/td/client/result_api_spec.rb +23 -0
- data/spec/td/client/sched_api_spec.rb +33 -0
- data/spec/td/client/server_status_api_spec.rb +25 -0
- data/spec/td/client/table_api_spec.rb +45 -0
- data/spec/td/client/user_api_spec.rb +118 -0
- data/spec/td/client_sched_spec.rb +54 -0
- metadata +37 -4
@@ -0,0 +1,62 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module AccessControl
|
3
|
+
|
4
|
+
####
|
5
|
+
## Access Control API
|
6
|
+
##
|
7
|
+
|
8
|
+
def grant_access_control(subject, action, scope, grant_option)
|
9
|
+
params = {'subject'=>subject, 'action'=>action, 'scope'=>scope, 'grant_option'=>grant_option.to_s}
|
10
|
+
code, body, res = post("/v3/acl/grant", params)
|
11
|
+
if code != "200"
|
12
|
+
raise_error("Granting access control failed", res)
|
13
|
+
end
|
14
|
+
return true
|
15
|
+
end
|
16
|
+
|
17
|
+
def revoke_access_control(subject, action, scope)
|
18
|
+
params = {'subject'=>subject, 'action'=>action, 'scope'=>scope}
|
19
|
+
code, body, res = post("/v3/acl/revoke", params)
|
20
|
+
if code != "200"
|
21
|
+
raise_error("Revoking access control failed", res)
|
22
|
+
end
|
23
|
+
return true
|
24
|
+
end
|
25
|
+
|
26
|
+
# [true, [{subject:String,action:String,scope:String}]]
|
27
|
+
def test_access_control(user, action, scope)
|
28
|
+
params = {'user'=>user, 'action'=>action, 'scope'=>scope}
|
29
|
+
code, body, res = get("/v3/acl/test", params)
|
30
|
+
if code != "200"
|
31
|
+
raise_error("Testing access control failed", res)
|
32
|
+
end
|
33
|
+
js = checked_json(body, %w[permission access_controls])
|
34
|
+
perm = js["permission"]
|
35
|
+
acl = js["access_controls"].map {|roleinfo|
|
36
|
+
subject = roleinfo['subject']
|
37
|
+
action = roleinfo['action']
|
38
|
+
scope = roleinfo['scope']
|
39
|
+
[name, action, scope]
|
40
|
+
}
|
41
|
+
return perm, acl
|
42
|
+
end
|
43
|
+
|
44
|
+
# [{subject:String,action:String,scope:String}]
|
45
|
+
def list_access_controls
|
46
|
+
code, body, res = get("/v3/acl/list")
|
47
|
+
if code != "200"
|
48
|
+
raise_error("Listing access control failed", res)
|
49
|
+
end
|
50
|
+
js = checked_json(body, %w[access_controls])
|
51
|
+
acl = js["access_controls"].map {|roleinfo|
|
52
|
+
subject = roleinfo['subject']
|
53
|
+
action = roleinfo['action']
|
54
|
+
scope = roleinfo['scope']
|
55
|
+
grant_option = roleinfo['grant_option']
|
56
|
+
[subject, action, scope, grant_option]
|
57
|
+
}
|
58
|
+
return acl
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module Account
|
3
|
+
|
4
|
+
####
|
5
|
+
## Account API
|
6
|
+
##
|
7
|
+
|
8
|
+
def show_account
|
9
|
+
code, body, res = get("/v3/account/show")
|
10
|
+
if code != "200"
|
11
|
+
raise_error("Show account failed", res)
|
12
|
+
end
|
13
|
+
js = checked_json(body, %w[account])
|
14
|
+
a = js["account"]
|
15
|
+
account_id = a['id'].to_i
|
16
|
+
plan = a['plan'].to_i
|
17
|
+
storage_size = a['storage_size'].to_i
|
18
|
+
guaranteed_cores = a['guaranteed_cores'].to_i
|
19
|
+
maximum_cores = a['maximum_cores'].to_i
|
20
|
+
created_at = a['created_at']
|
21
|
+
return [account_id, plan, storage_size, guaranteed_cores, maximum_cores, created_at]
|
22
|
+
end
|
23
|
+
|
24
|
+
def account_core_utilization(from, to)
|
25
|
+
params = { }
|
26
|
+
params['from'] = from.to_s if from
|
27
|
+
params['to'] = to.to_s if to
|
28
|
+
code, body, res = get("/v3/account/core_utilization", params)
|
29
|
+
if code != "200"
|
30
|
+
raise_error("Show account failed", res)
|
31
|
+
end
|
32
|
+
js = checked_json(body, %w[from to interval history])
|
33
|
+
from = Time.parse(js['from']).utc
|
34
|
+
to = Time.parse(js['to']).utc
|
35
|
+
interval = js['interval'].to_i
|
36
|
+
history = js['history']
|
37
|
+
return [from, to, interval, history]
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module BulkImport
|
3
|
+
|
4
|
+
####
|
5
|
+
## Bulk import API
|
6
|
+
##
|
7
|
+
|
8
|
+
# => nil
|
9
|
+
def create_bulk_import(name, db, table, opts={})
|
10
|
+
params = opts.dup
|
11
|
+
code, body, res = post("/v3/bulk_import/create/#{e name}/#{e db}/#{e table}", params)
|
12
|
+
if code != "200"
|
13
|
+
raise_error("Create bulk import failed", res)
|
14
|
+
end
|
15
|
+
return nil
|
16
|
+
end
|
17
|
+
|
18
|
+
# => nil
|
19
|
+
def delete_bulk_import(name, opts={})
|
20
|
+
params = opts.dup
|
21
|
+
code, body, res = post("/v3/bulk_import/delete/#{e name}", params)
|
22
|
+
if code != "200"
|
23
|
+
raise_error("Delete bulk import failed", res)
|
24
|
+
end
|
25
|
+
return nil
|
26
|
+
end
|
27
|
+
|
28
|
+
# => data:Hash
|
29
|
+
def show_bulk_import(name)
|
30
|
+
code, body, res = get("/v3/bulk_import/show/#{name}")
|
31
|
+
if code != "200"
|
32
|
+
raise_error("Show bulk import failed", res)
|
33
|
+
end
|
34
|
+
js = checked_json(body, %w[status])
|
35
|
+
return js
|
36
|
+
end
|
37
|
+
|
38
|
+
# => result:[data:Hash]
|
39
|
+
def list_bulk_imports(opts={})
|
40
|
+
params = opts.dup
|
41
|
+
code, body, res = get("/v3/bulk_import/list", params)
|
42
|
+
if code != "200"
|
43
|
+
raise_error("List bulk imports failed", res)
|
44
|
+
end
|
45
|
+
js = checked_json(body, %w[bulk_imports])
|
46
|
+
return js['bulk_imports']
|
47
|
+
end
|
48
|
+
|
49
|
+
def list_bulk_import_parts(name, opts={})
|
50
|
+
params = opts.dup
|
51
|
+
code, body, res = get("/v3/bulk_import/list_parts/#{e name}", params)
|
52
|
+
if code != "200"
|
53
|
+
raise_error("List bulk import parts failed", res)
|
54
|
+
end
|
55
|
+
js = checked_json(body, %w[parts])
|
56
|
+
return js['parts']
|
57
|
+
end
|
58
|
+
|
59
|
+
# => nil
|
60
|
+
def bulk_import_upload_part(name, part_name, stream, size, opts={})
|
61
|
+
code, body, res = put("/v3/bulk_import/upload_part/#{e name}/#{e part_name}", stream, size)
|
62
|
+
if code[0] != ?2
|
63
|
+
raise_error("Upload a part failed", res)
|
64
|
+
end
|
65
|
+
return nil
|
66
|
+
end
|
67
|
+
|
68
|
+
# => nil
|
69
|
+
def bulk_import_delete_part(name, part_name, opts={})
|
70
|
+
params = opts.dup
|
71
|
+
code, body, res = post("/v3/bulk_import/delete_part/#{e name}/#{e part_name}", params)
|
72
|
+
if code[0] != ?2
|
73
|
+
raise_error("Delete a part failed", res)
|
74
|
+
end
|
75
|
+
return nil
|
76
|
+
end
|
77
|
+
|
78
|
+
# => nil
|
79
|
+
def freeze_bulk_import(name, opts={})
|
80
|
+
params = opts.dup
|
81
|
+
code, body, res = post("/v3/bulk_import/freeze/#{e name}", params)
|
82
|
+
if code != "200"
|
83
|
+
raise_error("Freeze bulk import failed", res)
|
84
|
+
end
|
85
|
+
return nil
|
86
|
+
end
|
87
|
+
|
88
|
+
# => nil
|
89
|
+
def unfreeze_bulk_import(name, opts={})
|
90
|
+
params = opts.dup
|
91
|
+
code, body, res = post("/v3/bulk_import/unfreeze/#{e name}", params)
|
92
|
+
if code != "200"
|
93
|
+
raise_error("Unfreeze bulk import failed", res)
|
94
|
+
end
|
95
|
+
return nil
|
96
|
+
end
|
97
|
+
|
98
|
+
# => jobId:String
|
99
|
+
def perform_bulk_import(name, opts={})
|
100
|
+
params = opts.dup
|
101
|
+
code, body, res = post("/v3/bulk_import/perform/#{e name}", params)
|
102
|
+
if code != "200"
|
103
|
+
raise_error("Perform bulk import failed", res)
|
104
|
+
end
|
105
|
+
js = checked_json(body, %w[job_id])
|
106
|
+
return js['job_id'].to_s
|
107
|
+
end
|
108
|
+
|
109
|
+
# => nil
|
110
|
+
def commit_bulk_import(name, opts={})
|
111
|
+
params = opts.dup
|
112
|
+
code, body, res = post("/v3/bulk_import/commit/#{e name}", params)
|
113
|
+
if code != "200"
|
114
|
+
raise_error("Commit bulk import failed", res)
|
115
|
+
end
|
116
|
+
return nil
|
117
|
+
end
|
118
|
+
|
119
|
+
# => data...
|
120
|
+
def bulk_import_error_records(name, opts={}, &block)
|
121
|
+
params = opts.dup
|
122
|
+
code, body, res = get("/v3/bulk_import/error_records/#{e name}", params)
|
123
|
+
if code != "200"
|
124
|
+
raise_error("Failed to get bulk import error records", res)
|
125
|
+
end
|
126
|
+
if body.nil? || body.empty?
|
127
|
+
if block
|
128
|
+
return nil
|
129
|
+
else
|
130
|
+
return []
|
131
|
+
end
|
132
|
+
end
|
133
|
+
require File.expand_path('../compat_gzip_reader', File.dirname(__FILE__))
|
134
|
+
u = MessagePack::Unpacker.new(Zlib::GzipReader.new(StringIO.new(body)))
|
135
|
+
if block
|
136
|
+
begin
|
137
|
+
u.each(&block)
|
138
|
+
rescue EOFError
|
139
|
+
end
|
140
|
+
nil
|
141
|
+
else
|
142
|
+
result = []
|
143
|
+
begin
|
144
|
+
u.each {|row|
|
145
|
+
result << row
|
146
|
+
}
|
147
|
+
rescue EOFError
|
148
|
+
end
|
149
|
+
return result
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
end
|
154
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module Database
|
3
|
+
|
4
|
+
####
|
5
|
+
## Database API
|
6
|
+
##
|
7
|
+
|
8
|
+
# => [name:String]
|
9
|
+
def list_databases
|
10
|
+
code, body, res = get("/v3/database/list")
|
11
|
+
if code != "200"
|
12
|
+
raise_error("List databases failed", res)
|
13
|
+
end
|
14
|
+
js = checked_json(body, %w[databases])
|
15
|
+
result = {}
|
16
|
+
js["databases"].each {|m|
|
17
|
+
name = m['name']
|
18
|
+
count = m['count']
|
19
|
+
created_at = m['created_at']
|
20
|
+
updated_at = m['updated_at']
|
21
|
+
permission = m['permission']
|
22
|
+
result[name] = [count, created_at, updated_at, nil, permission] # set nil to org for API compatibiilty
|
23
|
+
}
|
24
|
+
return result
|
25
|
+
end
|
26
|
+
|
27
|
+
# => true
|
28
|
+
def delete_database(db)
|
29
|
+
code, body, res = post("/v3/database/delete/#{e db}")
|
30
|
+
if code != "200"
|
31
|
+
raise_error("Delete database failed", res)
|
32
|
+
end
|
33
|
+
return true
|
34
|
+
end
|
35
|
+
|
36
|
+
# => true
|
37
|
+
def create_database(db, opts={})
|
38
|
+
params = opts.dup
|
39
|
+
code, body, res = post("/v3/database/create/#{e db}", params)
|
40
|
+
if code != "200"
|
41
|
+
raise_error("Create database failed", res)
|
42
|
+
end
|
43
|
+
return true
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module Export
|
3
|
+
|
4
|
+
####
|
5
|
+
## Export API
|
6
|
+
##
|
7
|
+
|
8
|
+
# => jobId:String
|
9
|
+
def export(db, table, storage_type, opts={})
|
10
|
+
params = opts.dup
|
11
|
+
params['storage_type'] = storage_type
|
12
|
+
code, body, res = post("/v3/export/run/#{e db}/#{e table}", params)
|
13
|
+
if code != "200"
|
14
|
+
raise_error("Export failed", res)
|
15
|
+
end
|
16
|
+
js = checked_json(body, %w[job_id])
|
17
|
+
return js['job_id'].to_s
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module Import
|
3
|
+
|
4
|
+
####
|
5
|
+
## Import API
|
6
|
+
##
|
7
|
+
|
8
|
+
# => time:Float
|
9
|
+
def import(db, table, format, stream, size, unique_id=nil)
|
10
|
+
if unique_id
|
11
|
+
path = "/v3/table/import_with_id/#{e db}/#{e table}/#{unique_id}/#{format}"
|
12
|
+
else
|
13
|
+
path = "/v3/table/import/#{e db}/#{e table}/#{format}"
|
14
|
+
end
|
15
|
+
opts = {}
|
16
|
+
if @host == DEFAULT_ENDPOINT
|
17
|
+
opts[:host] = DEFAULT_IMPORT_ENDPOINT
|
18
|
+
elsif @host == NEW_DEFAULT_ENDPOINT
|
19
|
+
opts[:host] = NEW_DEFAULT_IMPORT_ENDPOINT
|
20
|
+
end
|
21
|
+
code, body, res = put(path, stream, size, opts)
|
22
|
+
if code[0] != ?2
|
23
|
+
raise_error("Import failed", res)
|
24
|
+
end
|
25
|
+
js = checked_json(body, %w[])
|
26
|
+
time = js['elapsed_time'].to_f
|
27
|
+
return time
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,251 @@
|
|
1
|
+
class TreasureData::API
|
2
|
+
module Job
|
3
|
+
|
4
|
+
####
|
5
|
+
## Job API
|
6
|
+
##
|
7
|
+
|
8
|
+
# => [(jobId:String, type:Symbol, status:String, start_at:String, end_at:String, result_url:String)]
|
9
|
+
def list_jobs(from=0, to=nil, status=nil, conditions=nil)
|
10
|
+
params = {}
|
11
|
+
params['from'] = from.to_s if from
|
12
|
+
params['to'] = to.to_s if to
|
13
|
+
params['status'] = status.to_s if status
|
14
|
+
params.merge!(conditions) if conditions
|
15
|
+
code, body, res = get("/v3/job/list", params)
|
16
|
+
if code != "200"
|
17
|
+
raise_error("List jobs failed", res)
|
18
|
+
end
|
19
|
+
js = checked_json(body, %w[jobs])
|
20
|
+
result = []
|
21
|
+
js['jobs'].each {|m|
|
22
|
+
job_id = m['job_id']
|
23
|
+
type = (m['type'] || '?').to_sym
|
24
|
+
database = m['database']
|
25
|
+
status = m['status']
|
26
|
+
query = m['query']
|
27
|
+
start_at = m['start_at']
|
28
|
+
end_at = m['end_at']
|
29
|
+
cpu_time = m['cpu_time']
|
30
|
+
result_size = m['result_size'] # compressed result size in msgpack.gz format
|
31
|
+
result_url = m['result']
|
32
|
+
priority = m['priority']
|
33
|
+
retry_limit = m['retry_limit']
|
34
|
+
result << [job_id, type, status, query, start_at, end_at, cpu_time,
|
35
|
+
result_size, result_url, priority, retry_limit, nil, database]
|
36
|
+
}
|
37
|
+
return result
|
38
|
+
end
|
39
|
+
|
40
|
+
# => (type:Symbol, status:String, result:String, url:String, result:String)
|
41
|
+
def show_job(job_id)
|
42
|
+
# use v3/job/status instead of v3/job/show to poll finish of a job
|
43
|
+
code, body, res = get("/v3/job/show/#{e job_id}")
|
44
|
+
if code != "200"
|
45
|
+
raise_error("Show job failed", res)
|
46
|
+
end
|
47
|
+
js = checked_json(body, %w[status])
|
48
|
+
# TODO debug
|
49
|
+
type = (js['type'] || '?').to_sym # TODO
|
50
|
+
database = js['database']
|
51
|
+
query = js['query']
|
52
|
+
status = js['status']
|
53
|
+
debug = js['debug']
|
54
|
+
url = js['url']
|
55
|
+
start_at = js['start_at']
|
56
|
+
end_at = js['end_at']
|
57
|
+
cpu_time = js['cpu_time']
|
58
|
+
result_size = js['result_size'] # compressed result size in msgpack.gz format
|
59
|
+
result = js['result'] # result target URL
|
60
|
+
hive_result_schema = (js['hive_result_schema'] || '')
|
61
|
+
if hive_result_schema.empty?
|
62
|
+
hive_result_schema = nil
|
63
|
+
else
|
64
|
+
begin
|
65
|
+
hive_result_schema = JSON.parse(hive_result_schema)
|
66
|
+
rescue JSON::ParserError => e
|
67
|
+
# this is a workaround for a Known Limitation in the Pig Engine which does not set a default, auto-generated
|
68
|
+
# column name for anonymous columns (such as the ones that are generated from UDF like COUNT or SUM).
|
69
|
+
# The schema will contain 'nil' for the name of those columns and that breaks the JSON parser since it violates
|
70
|
+
# the JSON syntax standard.
|
71
|
+
if type == :pig and hive_result_schema !~ /[\{\}]/
|
72
|
+
begin
|
73
|
+
# NOTE: this works because a JSON 2 dimensional array is the same as a Ruby one.
|
74
|
+
# Any change in the format for the hive_result_schema output may cause a syntax error, in which case
|
75
|
+
# this lame attempt at fixing the problem will fail and we will be raising the original JSON exception
|
76
|
+
hive_result_schema = eval(hive_result_schema)
|
77
|
+
rescue SyntaxError => ignored_e
|
78
|
+
raise e
|
79
|
+
end
|
80
|
+
hive_result_schema.each_with_index {|col_schema, idx|
|
81
|
+
if col_schema[0].nil?
|
82
|
+
col_schema[0] = "_col#{idx}"
|
83
|
+
end
|
84
|
+
}
|
85
|
+
else
|
86
|
+
raise e
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
priority = js['priority']
|
91
|
+
retry_limit = js['retry_limit']
|
92
|
+
return [type, query, status, url, debug, start_at, end_at, cpu_time,
|
93
|
+
result_size, result, hive_result_schema, priority, retry_limit, nil, database]
|
94
|
+
end
|
95
|
+
|
96
|
+
def job_status(job_id)
|
97
|
+
code, body, res = get("/v3/job/status/#{e job_id}")
|
98
|
+
if code != "200"
|
99
|
+
raise_error("Get job status failed", res)
|
100
|
+
end
|
101
|
+
|
102
|
+
js = checked_json(body, %w[status])
|
103
|
+
return js['status']
|
104
|
+
end
|
105
|
+
|
106
|
+
def job_result(job_id)
|
107
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
|
108
|
+
if code != "200"
|
109
|
+
raise_error("Get job result failed", res)
|
110
|
+
end
|
111
|
+
result = []
|
112
|
+
MessagePack::Unpacker.new.feed_each(body) {|row|
|
113
|
+
result << row
|
114
|
+
}
|
115
|
+
return result
|
116
|
+
end
|
117
|
+
|
118
|
+
# block is optional and must accept 1 parameter
|
119
|
+
def job_result_format(job_id, format, io=nil, &block)
|
120
|
+
if io
|
121
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res|
|
122
|
+
if res.code != "200"
|
123
|
+
raise_error("Get job result failed", res)
|
124
|
+
end
|
125
|
+
|
126
|
+
if ce = res.header['Content-Encoding']
|
127
|
+
res.extend(DeflateReadBodyMixin)
|
128
|
+
res.gzip = true if ce == 'gzip'
|
129
|
+
else
|
130
|
+
res.extend(DirectReadBodyMixin)
|
131
|
+
end
|
132
|
+
|
133
|
+
res.extend(DirectReadBodyMixin)
|
134
|
+
if ce = res.header['Content-Encoding']
|
135
|
+
if ce == 'gzip'
|
136
|
+
infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
|
137
|
+
else
|
138
|
+
infl = Zlib::Inflate.new
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
total_compr_size = 0
|
143
|
+
res.each_fragment {|fragment|
|
144
|
+
total_compr_size += fragment.size
|
145
|
+
# uncompressed if the 'Content-Enconding' header is set in response
|
146
|
+
fragment = infl.inflate(fragment) if ce
|
147
|
+
io.write(fragment)
|
148
|
+
block.call(total_compr_size) if block_given?
|
149
|
+
}
|
150
|
+
}
|
151
|
+
nil
|
152
|
+
else
|
153
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
|
154
|
+
if res.code != "200"
|
155
|
+
raise_error("Get job result failed", res)
|
156
|
+
end
|
157
|
+
body
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
# block is optional and must accept 1 argument
|
162
|
+
def job_result_each(job_id, &block)
|
163
|
+
get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
|
164
|
+
if res.code != "200"
|
165
|
+
raise_error("Get job result failed", res)
|
166
|
+
end
|
167
|
+
|
168
|
+
# default to decompressing the response since format is fixed to 'msgpack'
|
169
|
+
res.extend(DeflateReadBodyMixin)
|
170
|
+
res.gzip = (res.header['Content-Encoding'] == 'gzip')
|
171
|
+
upkr = MessagePack::Unpacker.new
|
172
|
+
res.each_fragment {|inflated_fragment|
|
173
|
+
upkr.feed_each(inflated_fragment, &block)
|
174
|
+
}
|
175
|
+
}
|
176
|
+
nil
|
177
|
+
end
|
178
|
+
|
179
|
+
# block is optional and must accept 1 argument
|
180
|
+
def job_result_each_with_compr_size(job_id, &block)
|
181
|
+
get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
|
182
|
+
if res.code != "200"
|
183
|
+
raise_error("Get job result failed", res)
|
184
|
+
end
|
185
|
+
|
186
|
+
res.extend(DirectReadBodyMixin)
|
187
|
+
if res.header['Content-Encoding'] == 'gzip'
|
188
|
+
infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
|
189
|
+
else
|
190
|
+
infl = Zlib::Inflate.new
|
191
|
+
end
|
192
|
+
upkr = MessagePack::Unpacker.new
|
193
|
+
begin
|
194
|
+
total_compr_size = 0
|
195
|
+
res.each_fragment {|fragment|
|
196
|
+
total_compr_size += fragment.size
|
197
|
+
upkr.feed_each(infl.inflate(fragment)) {|unpacked|
|
198
|
+
block.call(unpacked, total_compr_size) if block_given?
|
199
|
+
}
|
200
|
+
}
|
201
|
+
ensure
|
202
|
+
infl.close
|
203
|
+
end
|
204
|
+
}
|
205
|
+
nil
|
206
|
+
end
|
207
|
+
|
208
|
+
def job_result_raw(job_id, format)
|
209
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
|
210
|
+
if code != "200"
|
211
|
+
raise_error("Get job result failed", res)
|
212
|
+
end
|
213
|
+
return body
|
214
|
+
end
|
215
|
+
|
216
|
+
def kill(job_id)
|
217
|
+
code, body, res = post("/v3/job/kill/#{e job_id}")
|
218
|
+
if code != "200"
|
219
|
+
raise_error("Kill job failed", res)
|
220
|
+
end
|
221
|
+
js = checked_json(body, %w[])
|
222
|
+
former_status = js['former_status']
|
223
|
+
return former_status
|
224
|
+
end
|
225
|
+
|
226
|
+
# => jobId:String
|
227
|
+
def hive_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
|
228
|
+
query(q, :hive, db, result_url, priority, retry_limit, opts)
|
229
|
+
end
|
230
|
+
|
231
|
+
# => jobId:String
|
232
|
+
def pig_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
|
233
|
+
query(q, :pig, db, result_url, priority, retry_limit, opts)
|
234
|
+
end
|
235
|
+
|
236
|
+
# => jobId:String
|
237
|
+
def query(q, type=:hive, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
|
238
|
+
params = {'query' => q}.merge(opts)
|
239
|
+
params['result'] = result_url if result_url
|
240
|
+
params['priority'] = priority if priority
|
241
|
+
params['retry_limit'] = retry_limit if retry_limit
|
242
|
+
code, body, res = post("/v3/job/issue/#{type}/#{e db}", params)
|
243
|
+
if code != "200"
|
244
|
+
raise_error("Query failed", res)
|
245
|
+
end
|
246
|
+
js = checked_json(body, %w[job_id])
|
247
|
+
return js['job_id'].to_s
|
248
|
+
end
|
249
|
+
|
250
|
+
end
|
251
|
+
end
|