aliyun-odps 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.rubocop.yml +31 -0
  4. data/Gemfile +3 -0
  5. data/README.md +55 -12
  6. data/Rakefile +15 -5
  7. data/aliyun-odps.gemspec +22 -11
  8. data/bin/console +10 -3
  9. data/lib/aliyun/odps.rb +69 -2
  10. data/lib/aliyun/odps/authorization.rb +90 -0
  11. data/lib/aliyun/odps/client.rb +40 -0
  12. data/lib/aliyun/odps/configuration.rb +16 -0
  13. data/lib/aliyun/odps/error.rb +97 -0
  14. data/lib/aliyun/odps/http.rb +138 -0
  15. data/lib/aliyun/odps/list.rb +40 -0
  16. data/lib/aliyun/odps/model/function.rb +16 -0
  17. data/lib/aliyun/odps/model/functions.rb +113 -0
  18. data/lib/aliyun/odps/model/instance.rb +130 -0
  19. data/lib/aliyun/odps/model/instance_task.rb +30 -0
  20. data/lib/aliyun/odps/model/instances.rb +119 -0
  21. data/lib/aliyun/odps/model/projects.rb +73 -0
  22. data/lib/aliyun/odps/model/resource.rb +26 -0
  23. data/lib/aliyun/odps/model/resources.rb +144 -0
  24. data/lib/aliyun/odps/model/table.rb +37 -0
  25. data/lib/aliyun/odps/model/table_column.rb +13 -0
  26. data/lib/aliyun/odps/model/table_partition.rb +9 -0
  27. data/lib/aliyun/odps/model/table_partitions.rb +90 -0
  28. data/lib/aliyun/odps/model/table_schema.rb +13 -0
  29. data/lib/aliyun/odps/model/tables.rb +125 -0
  30. data/lib/aliyun/odps/model/task_result.rb +9 -0
  31. data/lib/aliyun/odps/modelable.rb +16 -0
  32. data/lib/aliyun/odps/project.rb +47 -0
  33. data/lib/aliyun/odps/service_object.rb +27 -0
  34. data/lib/aliyun/odps/struct.rb +126 -0
  35. data/lib/aliyun/odps/tunnel/download_session.rb +98 -0
  36. data/lib/aliyun/odps/tunnel/router.rb +15 -0
  37. data/lib/aliyun/odps/tunnel/snappy_reader.rb +19 -0
  38. data/lib/aliyun/odps/tunnel/snappy_writer.rb +45 -0
  39. data/lib/aliyun/odps/tunnel/table_tunnels.rb +81 -0
  40. data/lib/aliyun/odps/tunnel/upload_block.rb +9 -0
  41. data/lib/aliyun/odps/tunnel/upload_session.rb +132 -0
  42. data/lib/aliyun/odps/utils.rb +102 -0
  43. data/lib/aliyun/odps/version.rb +1 -1
  44. data/requirements.png +0 -0
  45. data/wiki/error.md +188 -0
  46. data/wiki/functions.md +39 -0
  47. data/wiki/get_start.md +34 -0
  48. data/wiki/installation.md +15 -0
  49. data/wiki/instances.md +32 -0
  50. data/wiki/projects.md +51 -0
  51. data/wiki/resources.md +62 -0
  52. data/wiki/ssl.md +7 -0
  53. data/wiki/tables.md +75 -0
  54. data/wiki/tunnels.md +80 -0
  55. metadata +195 -13
  56. data/requirements.mindnode/QuickLook/Preview.jpg +0 -0
  57. data/requirements.mindnode/contents.xml +0 -10711
  58. data/requirements.mindnode/viewState.plist +0 -0
@@ -0,0 +1,30 @@
1
+ module Aliyun
2
+ module Odps
3
+ class InstanceTask < Struct::Base
4
+ property :name, String, required: true
5
+ property :type, String, required: true, within: %w(SQL SQLPLAN MapReduce DT PLSQL)
6
+ property :comment, String
7
+ property :property, Hash, init_with: ->(hash) do
8
+ hash.map { |k, v| { 'Name' => k, 'Value' => v } }
9
+ end
10
+ property :query, String
11
+ property :start_time, DateTime
12
+ property :end_time, DateTime
13
+ property :status, String
14
+ property :histories, Array
15
+
16
+ def to_hash
17
+ {
18
+ 'SQL' => {
19
+ 'Name' => name,
20
+ 'Comment' => comment || '',
21
+ 'Config' => {
22
+ 'Property' => property || { 'Name' => '', 'Value' => '' }
23
+ },
24
+ 'Query!' => "<![CDATA[#{query}]]>"
25
+ }
26
+ }
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,119 @@
1
+ module Aliyun
2
+ module Odps
3
+ class Instances < ServiceObject
4
+ DEFAULT_PRIORITY = 9
5
+
6
+ # List instances of project
7
+ #
8
+ # @see http://repo.aliyun.com/api-doc/Instance/get_instances/index.html Get instances
9
+ #
10
+ # @params options [Hash] options
11
+ #
12
+ # @option options [String] :datarange specify the starttime range time range
13
+ # @option options [String] :status supported value: Running, Suspended, Terminated
14
+ # @option jobname [String] :jobname specify the job name
15
+ # @option onlyowner [String] :onlyowner (yes) supported value: yes, no
16
+ # @option options [String] :marker
17
+ # @option options [String] :maxitems (1000)
18
+ #
19
+ # @return [List]
20
+ def list(options = {})
21
+ Utils.stringify_keys!(options)
22
+ path = "/projects/#{project.name}/instances"
23
+ query = Utils.hash_slice(options, 'datarange', 'status', 'jobname', 'onlyowner', 'marker', 'maxitems')
24
+ result = client.get(path, query: query).parsed_response
25
+
26
+ Aliyun::Odps::List.build(result, %w(Instances Instance)) do |hash|
27
+ Instance.new(hash.merge(project: project))
28
+ end
29
+ end
30
+
31
+ # Create a instance job
32
+ #
33
+ # @see http://repo.aliyun.com/api-doc/Instance/post_instance/index.html Post Instance
34
+ #
35
+ # @params tasks [Array<InstanceTask>] a list of instance_task
36
+ # @params options [String] options
37
+ # @option options [String] :name Specify the instance name
38
+ # @option options [String] :comment Specify comment of the instance
39
+ # @option options [Integer] :priority Specify priority of the instance
40
+ #
41
+ # @raise InstanceNameInvalidError if instance name not valid
42
+ #
43
+ # @return [Instance]
44
+ def create(tasks, options = {})
45
+ Utils.stringify_keys!(options)
46
+ path = "/projects/#{project.name}/instances"
47
+
48
+ instance = validate_and_build_instance(tasks, options)
49
+ resp = client.post(path, body: build_create_body(instance))
50
+
51
+ append_location(instance, resp.headers['Location'])
52
+ end
53
+
54
+ # Get status of instance
55
+ #
56
+ # @see http://repo.aliyun.com/api-doc/Instance/get_instance/index.html Get instance
57
+ #
58
+ # @params name [String] specify the instance name
59
+ #
60
+ # @return Instance status: Suspended, Running, Terminated
61
+ def status(name)
62
+ instance = Instance.new(
63
+ name: name,
64
+ project: project
65
+ )
66
+
67
+ instance.get_status
68
+ end
69
+
70
+ private
71
+
72
+ def build_create_body(instance)
73
+ fail XmlElementMissingError, 'Priority' if instance.priority.nil?
74
+ fail XmlElementMissingError, 'Tasks' if instance.tasks.empty?
75
+
76
+ Utils.to_xml(build_hash(instance), unwrap: true)
77
+ end
78
+
79
+ def build_hash(instance)
80
+ {
81
+ 'Instance' => {
82
+ 'Job' => {
83
+ 'Name' => instance.name,
84
+ 'Comment' => instance.comment || '',
85
+ 'Priority' => instance.priority,
86
+ 'Tasks' => instance.tasks.map(&:to_hash)
87
+ }
88
+ }
89
+ }
90
+ end
91
+
92
+ def validate_and_build_instance(tasks, options)
93
+ name = options.key?('name') ? options['name'] : Utils.generate_uuid('instance')
94
+ fail InstanceNameInvalidError, name unless name.match(Instance::NAME_PATTERN)
95
+ fail PriorityInvalidError if options.key?('priority') && options['priority'] < 0
96
+
97
+ build_instance(name, tasks, options['priority'], options['comment'])
98
+ end
99
+
100
+ def build_instance(name, tasks, priority = nil, comment = nil)
101
+ Instance.new(
102
+ name: name,
103
+ tasks: tasks,
104
+ project: project,
105
+ priority: priority || DEFAULT_PRIORITY,
106
+ client: client,
107
+ comment: comment
108
+ )
109
+ end
110
+
111
+ def append_location(instance, location)
112
+ instance.tap do |obj|
113
+ obj.location = location
114
+ obj.name = obj.location.split('/').last if obj.location
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,73 @@
1
+ module Aliyun
2
+ module Odps
3
+ # Methods for Projects
4
+ class Projects < ServiceObject
5
+ # List all projects
6
+ #
7
+ # @see http://repo.aliyun.com/api-doc/Project/get_projects/index.html Get projects
8
+ #
9
+ # @param options [Hash] options
10
+ # @option options [String] :owner specify the project owner
11
+ # @option options [String] :marker specify marker for paginate
12
+ # @option options [String] :maxitems (1000) specify maxitems in this request
13
+ #
14
+ # @return [List]
15
+ #
16
+ # TODO: http://git.oschina.net/newell_zlx/aliyun-odps-ruby-sdk/issues/2
17
+ def list(options = {})
18
+ Utils.stringify_keys!(options)
19
+ query = Utils.hash_slice(options, 'owner', 'marker', 'maxitems')
20
+ resp = client.get('/projects', query: query)
21
+ result = resp.parsed_response
22
+
23
+ Aliyun::Odps::List.build(result, %w(Projects Project)) do |hash|
24
+ Project.new(hash.merge(client: client))
25
+ end
26
+ end
27
+
28
+ # Get Project Information
29
+ #
30
+ # @see http://repo.aliyun.com/api-doc/Project/get_project/index.html Get Project
31
+ #
32
+ # @param name specify the project name
33
+ #
34
+ # @return [Project]
35
+ def get(name)
36
+ result = client.get("/projects/#{name}").parsed_response
37
+ hash = Utils.dig_value(result, 'Project')
38
+ Project.new(hash.merge(client: client))
39
+ end
40
+
41
+ # Update Project Information
42
+ #
43
+ # @see http://repo.aliyun.com/api-doc/Project/put_project/index.html Put Project
44
+ #
45
+ # @params name specify the project name
46
+ # @params options [Hash] options
47
+ # @option options [String] :comment Comment of the project
48
+ #
49
+ # @return true
50
+ def update(name, options = {})
51
+ Utils.stringify_keys!(options)
52
+ project = Project.new(
53
+ name: name,
54
+ comment: options['comment'],
55
+ client: client
56
+ )
57
+ !!client.put("/projects/#{name}", body: build_update_body(project))
58
+ end
59
+
60
+ private
61
+
62
+ def build_update_body(project)
63
+ fail XmlElementMissingError, 'Comment' if project.comment.nil?
64
+
65
+ Utils.to_xml(
66
+ 'Project' => {
67
+ 'Name' => project.name,
68
+ 'Comment' => project.comment
69
+ })
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,26 @@
1
+ module Aliyun
2
+ module Odps
3
+ class Resource < Struct::Base
4
+ extend Aliyun::Odps::Modelable
5
+
6
+ property :name, String, required: true
7
+ property :owner, String
8
+ property :last_updator, String
9
+ property :comment, String
10
+ property :resource_type, String, within: %w(py jar archive file table)
11
+ property :local_path, String
12
+ property :creation_time, DateTime
13
+ property :last_modified_time, DateTime
14
+ property :table_name, String
15
+ property :resource_size, Integer
16
+ property :content, String
17
+ property :location, String
18
+
19
+ alias_method :resource_name=, :name=
20
+
21
+ def to_hash
22
+ { 'ResourceName' => name }
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,144 @@
1
+ module Aliyun
2
+ module Odps
3
+ # Methods for Resources
4
+ class Resources < ServiceObject
5
+ # List resources of project
6
+ #
7
+ # @see http://repo.aliyun.com/api-doc/Resource/get_resources/index.html Get resources
8
+ #
9
+ # @params options [Hash] options
10
+ # @option options [String] :name specify resource name
11
+ # @option options [String] :owner specify resource owner
12
+ # @option options [String] :marker
13
+ # @option options [String] :maxitems (1000)
14
+ #
15
+ # @return [List]
16
+ def list(options = {})
17
+ Utils.stringify_keys!(options)
18
+ path = "/projects/#{project.name}/resources"
19
+ query = Utils.hash_slice(options, 'name', 'owner', 'marker', 'maxitems')
20
+ result = client.get(path, query: query).parsed_response
21
+
22
+ Aliyun::Odps::List.build(result, %w(Resources Resource)) do |hash|
23
+ Resource.new(hash)
24
+ end
25
+ end
26
+
27
+ # Get resource of project
28
+ #
29
+ # @see http://repo.aliyun.com/api-doc/Resource/get_resource/index.html Get resource
30
+ #
31
+ # @params name [String] specify resource name
32
+ #
33
+ # @return [Resource]
34
+ def get(name)
35
+ path = "/projects/#{project.name}/resources/#{name}"
36
+ resp = client.get(path)
37
+ build_resource(resp)
38
+ end
39
+ alias_method :resource, :get
40
+
41
+ # Get resource information in project
42
+ #
43
+ # @see http://repo.aliyun.com/api-doc/Resource/head_resource/index.html Head resource
44
+ #
45
+ # @params name [String] specify resource name
46
+ #
47
+ # @return [Resource]
48
+ def get_meta(name)
49
+ path = "/projects/#{project.name}/resources/#{name}"
50
+ resp = client.head(path)
51
+ build_resource(resp)
52
+ end
53
+ alias_method :head, :get_meta
54
+
55
+ # Create resource in project
56
+ #
57
+ # @see http://repo.aliyun.com/api-doc/Resource/post_resource/index.html Post resource
58
+ #
59
+ # @params name [String] specify resource name
60
+ # @params type [String] specify resource type: supported value: py, jar, archive, file, table
61
+ # @params options [Hash] options
62
+ # @option options [String] :comment specify resource comment
63
+ # @option options [String] :table specify table or table partition name, if your table have partitions, must contains partition spec together with format: tab1 partition(region='bj',year='2011')
64
+ # @option options [File|BinData] :file specify the source code or file path
65
+ #
66
+ # @return [Resource]
67
+ def create(name, type, options = {})
68
+ Utils.stringify_keys!(options)
69
+ path = "/projects/#{project.name}/resources/"
70
+
71
+ headers = build_create_base_headers(name, type, options)
72
+ body = build_create_base_body(options)
73
+
74
+ location = client.post(path, headers: headers, body: body).headers['Location']
75
+ Resource.new(name: name, resource_type: type, comment: options['comment'], location: location)
76
+ end
77
+
78
+ # Update resource in project
79
+ #
80
+ # @see http://repo.aliyun.com/api-doc/Resource/put_resource/index.html Put resource
81
+ #
82
+ # @params name [String] specify resource name
83
+ # @params type [String] specify resource type: supported value: py, jar, archive, file, table
84
+ # @params options [Hash] options
85
+ # @option options [String] :comment specify resource comment
86
+ # @option options [String] :table specify table or table partition name
87
+ # @option options [File|Bin Data] :file specify the source code or file path
88
+ #
89
+ # @return [true]
90
+ def update(name, type, options = {})
91
+ Utils.stringify_keys!(options)
92
+ path = "/projects/#{project.name}/resources/#{name}"
93
+
94
+ headers = build_create_base_headers(name, type, options)
95
+ body = build_create_base_body(options)
96
+
97
+ !!client.put(path, headers: headers, body: body)
98
+ end
99
+
100
+ # Delete resource in project
101
+ #
102
+ # @see http://repo.aliyun.com/api-doc/Resource/delete_resource/index.html Delete resource
103
+ #
104
+ # @params name [String] specify the resource name
105
+ #
106
+ # @return [true]
107
+ def delete(name)
108
+ path = "/projects/#{project.name}/resources/#{name}"
109
+
110
+ !!client.delete(path)
111
+ end
112
+
113
+ private
114
+
115
+ def build_create_base_headers(name, type, options)
116
+ headers = { 'x-odps-resource-type' => type, 'x-odps-resource-name' => name }
117
+ headers['x-odps-comment'] = options['comment'] if options.key?('comment')
118
+ headers['x-odps-copy-table-source'] = options['table'] if options.key?('table')
119
+ headers
120
+ end
121
+
122
+ def build_create_base_body(options)
123
+ body = options.key?('file') ? Utils.to_data(options['file']) : ''
124
+ fail ResourceMissingContentError if body.empty? && !options.key?('table')
125
+ body
126
+ end
127
+
128
+ def build_resource(resp)
129
+ hash = {
130
+ name: resp.headers['x-odps-resource-name'],
131
+ last_updator: resp.headers['x-odps-updator'],
132
+ owner: resp.headers['x-odps-owner'],
133
+ comment: resp.headers['x-odps-comment'],
134
+ last_modified_time: resp.headers['Last-Modified'],
135
+ creation_time: resp.headers['x-odps-creation-time'],
136
+ resource_size: resp.headers['x-odps-resource-size'],
137
+ resource_type: resp.headers['x-odps-resource-type']
138
+ }
139
+ hash['content'] = resp.parsed_response unless resp.parsed_response.nil?
140
+ Resource.new(hash)
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,37 @@
1
+ require 'aliyun/odps/model/table_partitions'
2
+ require 'aliyun/odps/model/table_schema'
3
+
4
+ module Aliyun
5
+ module Odps
6
+ class Table < Struct::Base
7
+ extend Aliyun::Odps::Modelable
8
+
9
+ # @!method table_partitions
10
+ # @return [TablePartitions]
11
+ has_many :table_partitions
12
+
13
+ property :project, Project, required: true
14
+
15
+ property :name, String, required: true
16
+ property :table_id, String
17
+ property :comment, String
18
+ property :owner, String
19
+ property :schema, TableSchema, init_with: ->(value) do
20
+ case value
21
+ when TableSchema
22
+ value
23
+ when Hash
24
+ value = JSON.parse(value['__content__']) if value.key?('__content__')
25
+ TableSchema.new(value)
26
+ end
27
+ end
28
+ property :creation_time, DateTime
29
+ property :last_modified, DateTime
30
+
31
+ # (see TablePartitions#list)
32
+ def partitions(options = {})
33
+ table_partitions.list(options)
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,13 @@
1
+ module Aliyun
2
+ module Odps
3
+ class TableColumn < Struct::Base
4
+ property :name, String, required: true
5
+ property :type, String, required: true, init_with: ->(value) do
6
+ fail NotSupportColumnTypeError, value unless %w(bigint double boolean datetime string).include?(value)
7
+ value
8
+ end
9
+ property :comment, String
10
+ property :label, String
11
+ end
12
+ end
13
+ end