aliyun-odps 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/.rubocop.yml +31 -0
- data/Gemfile +3 -0
- data/README.md +55 -12
- data/Rakefile +15 -5
- data/aliyun-odps.gemspec +22 -11
- data/bin/console +10 -3
- data/lib/aliyun/odps.rb +69 -2
- data/lib/aliyun/odps/authorization.rb +90 -0
- data/lib/aliyun/odps/client.rb +40 -0
- data/lib/aliyun/odps/configuration.rb +16 -0
- data/lib/aliyun/odps/error.rb +97 -0
- data/lib/aliyun/odps/http.rb +138 -0
- data/lib/aliyun/odps/list.rb +40 -0
- data/lib/aliyun/odps/model/function.rb +16 -0
- data/lib/aliyun/odps/model/functions.rb +113 -0
- data/lib/aliyun/odps/model/instance.rb +130 -0
- data/lib/aliyun/odps/model/instance_task.rb +30 -0
- data/lib/aliyun/odps/model/instances.rb +119 -0
- data/lib/aliyun/odps/model/projects.rb +73 -0
- data/lib/aliyun/odps/model/resource.rb +26 -0
- data/lib/aliyun/odps/model/resources.rb +144 -0
- data/lib/aliyun/odps/model/table.rb +37 -0
- data/lib/aliyun/odps/model/table_column.rb +13 -0
- data/lib/aliyun/odps/model/table_partition.rb +9 -0
- data/lib/aliyun/odps/model/table_partitions.rb +90 -0
- data/lib/aliyun/odps/model/table_schema.rb +13 -0
- data/lib/aliyun/odps/model/tables.rb +125 -0
- data/lib/aliyun/odps/model/task_result.rb +9 -0
- data/lib/aliyun/odps/modelable.rb +16 -0
- data/lib/aliyun/odps/project.rb +47 -0
- data/lib/aliyun/odps/service_object.rb +27 -0
- data/lib/aliyun/odps/struct.rb +126 -0
- data/lib/aliyun/odps/tunnel/download_session.rb +98 -0
- data/lib/aliyun/odps/tunnel/router.rb +15 -0
- data/lib/aliyun/odps/tunnel/snappy_reader.rb +19 -0
- data/lib/aliyun/odps/tunnel/snappy_writer.rb +45 -0
- data/lib/aliyun/odps/tunnel/table_tunnels.rb +81 -0
- data/lib/aliyun/odps/tunnel/upload_block.rb +9 -0
- data/lib/aliyun/odps/tunnel/upload_session.rb +132 -0
- data/lib/aliyun/odps/utils.rb +102 -0
- data/lib/aliyun/odps/version.rb +1 -1
- data/requirements.png +0 -0
- data/wiki/error.md +188 -0
- data/wiki/functions.md +39 -0
- data/wiki/get_start.md +34 -0
- data/wiki/installation.md +15 -0
- data/wiki/instances.md +32 -0
- data/wiki/projects.md +51 -0
- data/wiki/resources.md +62 -0
- data/wiki/ssl.md +7 -0
- data/wiki/tables.md +75 -0
- data/wiki/tunnels.md +80 -0
- metadata +195 -13
- data/requirements.mindnode/QuickLook/Preview.jpg +0 -0
- data/requirements.mindnode/contents.xml +0 -10711
- data/requirements.mindnode/viewState.plist +0 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
module Aliyun
|
2
|
+
module Odps
|
3
|
+
class InstanceTask < Struct::Base
|
4
|
+
property :name, String, required: true
|
5
|
+
property :type, String, required: true, within: %w(SQL SQLPLAN MapReduce DT PLSQL)
|
6
|
+
property :comment, String
|
7
|
+
property :property, Hash, init_with: ->(hash) do
|
8
|
+
hash.map { |k, v| { 'Name' => k, 'Value' => v } }
|
9
|
+
end
|
10
|
+
property :query, String
|
11
|
+
property :start_time, DateTime
|
12
|
+
property :end_time, DateTime
|
13
|
+
property :status, String
|
14
|
+
property :histories, Array
|
15
|
+
|
16
|
+
def to_hash
|
17
|
+
{
|
18
|
+
'SQL' => {
|
19
|
+
'Name' => name,
|
20
|
+
'Comment' => comment || '',
|
21
|
+
'Config' => {
|
22
|
+
'Property' => property || { 'Name' => '', 'Value' => '' }
|
23
|
+
},
|
24
|
+
'Query!' => "<![CDATA[#{query}]]>"
|
25
|
+
}
|
26
|
+
}
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module Aliyun
|
2
|
+
module Odps
|
3
|
+
class Instances < ServiceObject
|
4
|
+
DEFAULT_PRIORITY = 9
|
5
|
+
|
6
|
+
# List instances of project
|
7
|
+
#
|
8
|
+
# @see http://repo.aliyun.com/api-doc/Instance/get_instances/index.html Get instances
|
9
|
+
#
|
10
|
+
# @params options [Hash] options
|
11
|
+
#
|
12
|
+
# @option options [String] :datarange specify the starttime range time range
|
13
|
+
# @option options [String] :status supported value: Running, Suspended, Terminated
|
14
|
+
# @option jobname [String] :jobname specify the job name
|
15
|
+
# @option onlyowner [String] :onlyowner (yes) supported value: yes, no
|
16
|
+
# @option options [String] :marker
|
17
|
+
# @option options [String] :maxitems (1000)
|
18
|
+
#
|
19
|
+
# @return [List]
|
20
|
+
def list(options = {})
|
21
|
+
Utils.stringify_keys!(options)
|
22
|
+
path = "/projects/#{project.name}/instances"
|
23
|
+
query = Utils.hash_slice(options, 'datarange', 'status', 'jobname', 'onlyowner', 'marker', 'maxitems')
|
24
|
+
result = client.get(path, query: query).parsed_response
|
25
|
+
|
26
|
+
Aliyun::Odps::List.build(result, %w(Instances Instance)) do |hash|
|
27
|
+
Instance.new(hash.merge(project: project))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# Create a instance job
|
32
|
+
#
|
33
|
+
# @see http://repo.aliyun.com/api-doc/Instance/post_instance/index.html Post Instance
|
34
|
+
#
|
35
|
+
# @params tasks [Array<InstanceTask>] a list of instance_task
|
36
|
+
# @params options [String] options
|
37
|
+
# @option options [String] :name Specify the instance name
|
38
|
+
# @option options [String] :comment Specify comment of the instance
|
39
|
+
# @option options [Integer] :priority Specify priority of the instance
|
40
|
+
#
|
41
|
+
# @raise InstanceNameInvalidError if instance name not valid
|
42
|
+
#
|
43
|
+
# @return [Instance]
|
44
|
+
def create(tasks, options = {})
|
45
|
+
Utils.stringify_keys!(options)
|
46
|
+
path = "/projects/#{project.name}/instances"
|
47
|
+
|
48
|
+
instance = validate_and_build_instance(tasks, options)
|
49
|
+
resp = client.post(path, body: build_create_body(instance))
|
50
|
+
|
51
|
+
append_location(instance, resp.headers['Location'])
|
52
|
+
end
|
53
|
+
|
54
|
+
# Get status of instance
|
55
|
+
#
|
56
|
+
# @see http://repo.aliyun.com/api-doc/Instance/get_instance/index.html Get instance
|
57
|
+
#
|
58
|
+
# @params name [String] specify the instance name
|
59
|
+
#
|
60
|
+
# @return Instance status: Suspended, Running, Terminated
|
61
|
+
def status(name)
|
62
|
+
instance = Instance.new(
|
63
|
+
name: name,
|
64
|
+
project: project
|
65
|
+
)
|
66
|
+
|
67
|
+
instance.get_status
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def build_create_body(instance)
|
73
|
+
fail XmlElementMissingError, 'Priority' if instance.priority.nil?
|
74
|
+
fail XmlElementMissingError, 'Tasks' if instance.tasks.empty?
|
75
|
+
|
76
|
+
Utils.to_xml(build_hash(instance), unwrap: true)
|
77
|
+
end
|
78
|
+
|
79
|
+
def build_hash(instance)
|
80
|
+
{
|
81
|
+
'Instance' => {
|
82
|
+
'Job' => {
|
83
|
+
'Name' => instance.name,
|
84
|
+
'Comment' => instance.comment || '',
|
85
|
+
'Priority' => instance.priority,
|
86
|
+
'Tasks' => instance.tasks.map(&:to_hash)
|
87
|
+
}
|
88
|
+
}
|
89
|
+
}
|
90
|
+
end
|
91
|
+
|
92
|
+
def validate_and_build_instance(tasks, options)
|
93
|
+
name = options.key?('name') ? options['name'] : Utils.generate_uuid('instance')
|
94
|
+
fail InstanceNameInvalidError, name unless name.match(Instance::NAME_PATTERN)
|
95
|
+
fail PriorityInvalidError if options.key?('priority') && options['priority'] < 0
|
96
|
+
|
97
|
+
build_instance(name, tasks, options['priority'], options['comment'])
|
98
|
+
end
|
99
|
+
|
100
|
+
def build_instance(name, tasks, priority = nil, comment = nil)
|
101
|
+
Instance.new(
|
102
|
+
name: name,
|
103
|
+
tasks: tasks,
|
104
|
+
project: project,
|
105
|
+
priority: priority || DEFAULT_PRIORITY,
|
106
|
+
client: client,
|
107
|
+
comment: comment
|
108
|
+
)
|
109
|
+
end
|
110
|
+
|
111
|
+
def append_location(instance, location)
|
112
|
+
instance.tap do |obj|
|
113
|
+
obj.location = location
|
114
|
+
obj.name = obj.location.split('/').last if obj.location
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Aliyun
|
2
|
+
module Odps
|
3
|
+
# Methods for Projects
|
4
|
+
class Projects < ServiceObject
|
5
|
+
# List all projects
|
6
|
+
#
|
7
|
+
# @see http://repo.aliyun.com/api-doc/Project/get_projects/index.html Get projects
|
8
|
+
#
|
9
|
+
# @param options [Hash] options
|
10
|
+
# @option options [String] :owner specify the project owner
|
11
|
+
# @option options [String] :marker specify marker for paginate
|
12
|
+
# @option options [String] :maxitems (1000) specify maxitems in this request
|
13
|
+
#
|
14
|
+
# @return [List]
|
15
|
+
#
|
16
|
+
# TODO: http://git.oschina.net/newell_zlx/aliyun-odps-ruby-sdk/issues/2
|
17
|
+
def list(options = {})
|
18
|
+
Utils.stringify_keys!(options)
|
19
|
+
query = Utils.hash_slice(options, 'owner', 'marker', 'maxitems')
|
20
|
+
resp = client.get('/projects', query: query)
|
21
|
+
result = resp.parsed_response
|
22
|
+
|
23
|
+
Aliyun::Odps::List.build(result, %w(Projects Project)) do |hash|
|
24
|
+
Project.new(hash.merge(client: client))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# Get Project Information
|
29
|
+
#
|
30
|
+
# @see http://repo.aliyun.com/api-doc/Project/get_project/index.html Get Project
|
31
|
+
#
|
32
|
+
# @param name specify the project name
|
33
|
+
#
|
34
|
+
# @return [Project]
|
35
|
+
def get(name)
|
36
|
+
result = client.get("/projects/#{name}").parsed_response
|
37
|
+
hash = Utils.dig_value(result, 'Project')
|
38
|
+
Project.new(hash.merge(client: client))
|
39
|
+
end
|
40
|
+
|
41
|
+
# Update Project Information
|
42
|
+
#
|
43
|
+
# @see http://repo.aliyun.com/api-doc/Project/put_project/index.html Put Project
|
44
|
+
#
|
45
|
+
# @params name specify the project name
|
46
|
+
# @params options [Hash] options
|
47
|
+
# @option options [String] :comment Comment of the project
|
48
|
+
#
|
49
|
+
# @return true
|
50
|
+
def update(name, options = {})
|
51
|
+
Utils.stringify_keys!(options)
|
52
|
+
project = Project.new(
|
53
|
+
name: name,
|
54
|
+
comment: options['comment'],
|
55
|
+
client: client
|
56
|
+
)
|
57
|
+
!!client.put("/projects/#{name}", body: build_update_body(project))
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def build_update_body(project)
|
63
|
+
fail XmlElementMissingError, 'Comment' if project.comment.nil?
|
64
|
+
|
65
|
+
Utils.to_xml(
|
66
|
+
'Project' => {
|
67
|
+
'Name' => project.name,
|
68
|
+
'Comment' => project.comment
|
69
|
+
})
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Aliyun
|
2
|
+
module Odps
|
3
|
+
class Resource < Struct::Base
|
4
|
+
extend Aliyun::Odps::Modelable
|
5
|
+
|
6
|
+
property :name, String, required: true
|
7
|
+
property :owner, String
|
8
|
+
property :last_updator, String
|
9
|
+
property :comment, String
|
10
|
+
property :resource_type, String, within: %w(py jar archive file table)
|
11
|
+
property :local_path, String
|
12
|
+
property :creation_time, DateTime
|
13
|
+
property :last_modified_time, DateTime
|
14
|
+
property :table_name, String
|
15
|
+
property :resource_size, Integer
|
16
|
+
property :content, String
|
17
|
+
property :location, String
|
18
|
+
|
19
|
+
alias_method :resource_name=, :name=
|
20
|
+
|
21
|
+
def to_hash
|
22
|
+
{ 'ResourceName' => name }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
module Aliyun
|
2
|
+
module Odps
|
3
|
+
# Methods for Resources
|
4
|
+
class Resources < ServiceObject
|
5
|
+
# List resources of project
|
6
|
+
#
|
7
|
+
# @see http://repo.aliyun.com/api-doc/Resource/get_resources/index.html Get resources
|
8
|
+
#
|
9
|
+
# @params options [Hash] options
|
10
|
+
# @option options [String] :name specify resource name
|
11
|
+
# @option options [String] :owner specify resource owner
|
12
|
+
# @option options [String] :marker
|
13
|
+
# @option options [String] :maxitems (1000)
|
14
|
+
#
|
15
|
+
# @return [List]
|
16
|
+
def list(options = {})
|
17
|
+
Utils.stringify_keys!(options)
|
18
|
+
path = "/projects/#{project.name}/resources"
|
19
|
+
query = Utils.hash_slice(options, 'name', 'owner', 'marker', 'maxitems')
|
20
|
+
result = client.get(path, query: query).parsed_response
|
21
|
+
|
22
|
+
Aliyun::Odps::List.build(result, %w(Resources Resource)) do |hash|
|
23
|
+
Resource.new(hash)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# Get resource of project
|
28
|
+
#
|
29
|
+
# @see http://repo.aliyun.com/api-doc/Resource/get_resource/index.html Get resource
|
30
|
+
#
|
31
|
+
# @params name [String] specify resource name
|
32
|
+
#
|
33
|
+
# @return [Resource]
|
34
|
+
def get(name)
|
35
|
+
path = "/projects/#{project.name}/resources/#{name}"
|
36
|
+
resp = client.get(path)
|
37
|
+
build_resource(resp)
|
38
|
+
end
|
39
|
+
alias_method :resource, :get
|
40
|
+
|
41
|
+
# Get resource information in project
|
42
|
+
#
|
43
|
+
# @see http://repo.aliyun.com/api-doc/Resource/head_resource/index.html Head resource
|
44
|
+
#
|
45
|
+
# @params name [String] specify resource name
|
46
|
+
#
|
47
|
+
# @return [Resource]
|
48
|
+
def get_meta(name)
|
49
|
+
path = "/projects/#{project.name}/resources/#{name}"
|
50
|
+
resp = client.head(path)
|
51
|
+
build_resource(resp)
|
52
|
+
end
|
53
|
+
alias_method :head, :get_meta
|
54
|
+
|
55
|
+
# Create resource in project
|
56
|
+
#
|
57
|
+
# @see http://repo.aliyun.com/api-doc/Resource/post_resource/index.html Post resource
|
58
|
+
#
|
59
|
+
# @params name [String] specify resource name
|
60
|
+
# @params type [String] specify resource type: supported value: py, jar, archive, file, table
|
61
|
+
# @params options [Hash] options
|
62
|
+
# @option options [String] :comment specify resource comment
|
63
|
+
# @option options [String] :table specify table or table partition name, if your table have partitions, must contains partition spec together with format: tab1 partition(region='bj',year='2011')
|
64
|
+
# @option options [File|BinData] :file specify the source code or file path
|
65
|
+
#
|
66
|
+
# @return [Resource]
|
67
|
+
def create(name, type, options = {})
|
68
|
+
Utils.stringify_keys!(options)
|
69
|
+
path = "/projects/#{project.name}/resources/"
|
70
|
+
|
71
|
+
headers = build_create_base_headers(name, type, options)
|
72
|
+
body = build_create_base_body(options)
|
73
|
+
|
74
|
+
location = client.post(path, headers: headers, body: body).headers['Location']
|
75
|
+
Resource.new(name: name, resource_type: type, comment: options['comment'], location: location)
|
76
|
+
end
|
77
|
+
|
78
|
+
# Update resource in project
|
79
|
+
#
|
80
|
+
# @see http://repo.aliyun.com/api-doc/Resource/put_resource/index.html Put resource
|
81
|
+
#
|
82
|
+
# @params name [String] specify resource name
|
83
|
+
# @params type [String] specify resource type: supported value: py, jar, archive, file, table
|
84
|
+
# @params options [Hash] options
|
85
|
+
# @option options [String] :comment specify resource comment
|
86
|
+
# @option options [String] :table specify table or table partition name
|
87
|
+
# @option options [File|Bin Data] :file specify the source code or file path
|
88
|
+
#
|
89
|
+
# @return [true]
|
90
|
+
def update(name, type, options = {})
|
91
|
+
Utils.stringify_keys!(options)
|
92
|
+
path = "/projects/#{project.name}/resources/#{name}"
|
93
|
+
|
94
|
+
headers = build_create_base_headers(name, type, options)
|
95
|
+
body = build_create_base_body(options)
|
96
|
+
|
97
|
+
!!client.put(path, headers: headers, body: body)
|
98
|
+
end
|
99
|
+
|
100
|
+
# Delete resource in project
|
101
|
+
#
|
102
|
+
# @see http://repo.aliyun.com/api-doc/Resource/delete_resource/index.html Delete resource
|
103
|
+
#
|
104
|
+
# @params name [String] specify the resource name
|
105
|
+
#
|
106
|
+
# @return [true]
|
107
|
+
def delete(name)
|
108
|
+
path = "/projects/#{project.name}/resources/#{name}"
|
109
|
+
|
110
|
+
!!client.delete(path)
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
def build_create_base_headers(name, type, options)
|
116
|
+
headers = { 'x-odps-resource-type' => type, 'x-odps-resource-name' => name }
|
117
|
+
headers['x-odps-comment'] = options['comment'] if options.key?('comment')
|
118
|
+
headers['x-odps-copy-table-source'] = options['table'] if options.key?('table')
|
119
|
+
headers
|
120
|
+
end
|
121
|
+
|
122
|
+
def build_create_base_body(options)
|
123
|
+
body = options.key?('file') ? Utils.to_data(options['file']) : ''
|
124
|
+
fail ResourceMissingContentError if body.empty? && !options.key?('table')
|
125
|
+
body
|
126
|
+
end
|
127
|
+
|
128
|
+
def build_resource(resp)
|
129
|
+
hash = {
|
130
|
+
name: resp.headers['x-odps-resource-name'],
|
131
|
+
last_updator: resp.headers['x-odps-updator'],
|
132
|
+
owner: resp.headers['x-odps-owner'],
|
133
|
+
comment: resp.headers['x-odps-comment'],
|
134
|
+
last_modified_time: resp.headers['Last-Modified'],
|
135
|
+
creation_time: resp.headers['x-odps-creation-time'],
|
136
|
+
resource_size: resp.headers['x-odps-resource-size'],
|
137
|
+
resource_type: resp.headers['x-odps-resource-type']
|
138
|
+
}
|
139
|
+
hash['content'] = resp.parsed_response unless resp.parsed_response.nil?
|
140
|
+
Resource.new(hash)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'aliyun/odps/model/table_partitions'
|
2
|
+
require 'aliyun/odps/model/table_schema'
|
3
|
+
|
4
|
+
module Aliyun
|
5
|
+
module Odps
|
6
|
+
class Table < Struct::Base
|
7
|
+
extend Aliyun::Odps::Modelable
|
8
|
+
|
9
|
+
# @!method table_partitions
|
10
|
+
# @return [TablePartitions]
|
11
|
+
has_many :table_partitions
|
12
|
+
|
13
|
+
property :project, Project, required: true
|
14
|
+
|
15
|
+
property :name, String, required: true
|
16
|
+
property :table_id, String
|
17
|
+
property :comment, String
|
18
|
+
property :owner, String
|
19
|
+
property :schema, TableSchema, init_with: ->(value) do
|
20
|
+
case value
|
21
|
+
when TableSchema
|
22
|
+
value
|
23
|
+
when Hash
|
24
|
+
value = JSON.parse(value['__content__']) if value.key?('__content__')
|
25
|
+
TableSchema.new(value)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
property :creation_time, DateTime
|
29
|
+
property :last_modified, DateTime
|
30
|
+
|
31
|
+
# (see TablePartitions#list)
|
32
|
+
def partitions(options = {})
|
33
|
+
table_partitions.list(options)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Aliyun
|
2
|
+
module Odps
|
3
|
+
class TableColumn < Struct::Base
|
4
|
+
property :name, String, required: true
|
5
|
+
property :type, String, required: true, init_with: ->(value) do
|
6
|
+
fail NotSupportColumnTypeError, value unless %w(bigint double boolean datetime string).include?(value)
|
7
|
+
value
|
8
|
+
end
|
9
|
+
property :comment, String
|
10
|
+
property :label, String
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|