aliyun-oss-ruby-sdk 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +95 -0
  3. data/README.md +423 -0
  4. data/examples/aliyun/oss/bucket.rb +144 -0
  5. data/examples/aliyun/oss/callback.rb +61 -0
  6. data/examples/aliyun/oss/object.rb +182 -0
  7. data/examples/aliyun/oss/resumable_download.rb +42 -0
  8. data/examples/aliyun/oss/resumable_upload.rb +49 -0
  9. data/examples/aliyun/oss/streaming.rb +124 -0
  10. data/examples/aliyun/oss/using_sts.rb +48 -0
  11. data/examples/aliyun/sts/assume_role.rb +59 -0
  12. data/lib/aliyun_sdk/common.rb +6 -0
  13. data/lib/aliyun_sdk/common/exception.rb +18 -0
  14. data/lib/aliyun_sdk/common/logging.rb +46 -0
  15. data/lib/aliyun_sdk/common/struct.rb +56 -0
  16. data/lib/aliyun_sdk/oss.rb +16 -0
  17. data/lib/aliyun_sdk/oss/bucket.rb +661 -0
  18. data/lib/aliyun_sdk/oss/client.rb +106 -0
  19. data/lib/aliyun_sdk/oss/config.rb +39 -0
  20. data/lib/aliyun_sdk/oss/download.rb +255 -0
  21. data/lib/aliyun_sdk/oss/exception.rb +108 -0
  22. data/lib/aliyun_sdk/oss/http.rb +338 -0
  23. data/lib/aliyun_sdk/oss/iterator.rb +92 -0
  24. data/lib/aliyun_sdk/oss/multipart.rb +74 -0
  25. data/lib/aliyun_sdk/oss/object.rb +15 -0
  26. data/lib/aliyun_sdk/oss/protocol.rb +1499 -0
  27. data/lib/aliyun_sdk/oss/struct.rb +208 -0
  28. data/lib/aliyun_sdk/oss/upload.rb +238 -0
  29. data/lib/aliyun_sdk/oss/util.rb +89 -0
  30. data/lib/aliyun_sdk/sts.rb +9 -0
  31. data/lib/aliyun_sdk/sts/client.rb +38 -0
  32. data/lib/aliyun_sdk/sts/config.rb +22 -0
  33. data/lib/aliyun_sdk/sts/exception.rb +53 -0
  34. data/lib/aliyun_sdk/sts/protocol.rb +130 -0
  35. data/lib/aliyun_sdk/sts/struct.rb +64 -0
  36. data/lib/aliyun_sdk/sts/util.rb +48 -0
  37. data/lib/aliyun_sdk/version.rb +7 -0
  38. data/spec/aliyun/oss/bucket_spec.rb +597 -0
  39. data/spec/aliyun/oss/client/bucket_spec.rb +554 -0
  40. data/spec/aliyun/oss/client/client_spec.rb +297 -0
  41. data/spec/aliyun/oss/client/resumable_download_spec.rb +220 -0
  42. data/spec/aliyun/oss/client/resumable_upload_spec.rb +413 -0
  43. data/spec/aliyun/oss/http_spec.rb +83 -0
  44. data/spec/aliyun/oss/multipart_spec.rb +686 -0
  45. data/spec/aliyun/oss/object_spec.rb +785 -0
  46. data/spec/aliyun/oss/service_spec.rb +142 -0
  47. data/spec/aliyun/oss/util_spec.rb +50 -0
  48. data/spec/aliyun/sts/client_spec.rb +150 -0
  49. data/spec/aliyun/sts/util_spec.rb +39 -0
  50. data/tests/config.rb +31 -0
  51. data/tests/test_content_encoding.rb +54 -0
  52. data/tests/test_content_type.rb +95 -0
  53. data/tests/test_custom_headers.rb +70 -0
  54. data/tests/test_encoding.rb +77 -0
  55. data/tests/test_large_file.rb +66 -0
  56. data/tests/test_multipart.rb +97 -0
  57. data/tests/test_object_acl.rb +49 -0
  58. data/tests/test_object_key.rb +68 -0
  59. data/tests/test_object_url.rb +69 -0
  60. data/tests/test_resumable.rb +40 -0
  61. metadata +240 -0
@@ -0,0 +1,106 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module AliyunSDK
4
+ module OSS
5
+
6
+ ##
7
+ # OSS服务的客户端,用于获取bucket列表,创建/删除bucket。Object相关
8
+ # 的操作请使用{OSS::Bucket}。
9
+ # @example 创建Client
10
+ # endpoint = 'oss-cn-hangzhou.aliyuncs.com'
11
+ # client = Client.new(
12
+ # :endpoint => endpoint,
13
+ # :access_key_id => 'access_key_id',
14
+ # :access_key_secret => 'access_key_secret')
15
+ # buckets = client.list_buckets
16
+ # client.create_bucket('my-bucket')
17
+ # client.delete_bucket('my-bucket')
18
+ # bucket = client.get_bucket('my-bucket')
19
+ class Client
20
+
21
+ # 构造OSS client,用于操作buckets。
22
+ # @param opts [Hash] 构造Client时的参数选项
23
+ # @option opts [String] :endpoint [必填]OSS服务的地址,可以是以
24
+ # oss-cn-hangzhou.aliyuncs.com的标准域名,也可以是用户绑定的域名
25
+ # @option opts [String] :access_key_id [可选]用户的ACCESS KEY ID,
26
+ # 如果不填则会尝试匿名访问
27
+ # @option opts [String] :access_key_secret [可选]用户的ACCESS
28
+ # KEY SECRET,如果不填则会尝试匿名访问
29
+ # @option opts [Boolean] :cname [可选] 指定endpoint是否是用户绑
30
+ # 定的域名
31
+ # @option opts [String] :sts_token [可选] 指定STS的
32
+ # SecurityToken,如果指定,则使用STS授权访问
33
+ # @option opts [Fixnum] :open_timeout [可选] 指定建立连接的超时
34
+ # 时间,默认为10秒
35
+ # @option opts [Fixnum] :read_timeout [可选] 指定等待响应的超时
36
+ # 时间,默认为120秒
37
+ # @example 标准endpoint
38
+ # oss-cn-hangzhou.aliyuncs.com
39
+ # oss-cn-beijing.aliyuncs.com
40
+ # @example 用户绑定的域名
41
+ # my-domain.com
42
+ # foo.bar.com
43
+ def initialize(opts)
44
+ fail ClientError, "Endpoint must be provided" unless opts[:endpoint]
45
+
46
+ @config = Config.new(opts)
47
+ @protocol = Protocol.new(@config)
48
+ end
49
+
50
+ # 列出当前所有的bucket
51
+ # @param opts [Hash] 查询选项
52
+ # @option opts [String] :prefix 如果设置,则只返回以它为前缀的bucket
53
+ # @option opts [String] :marker 如果设置,则只返回名字在它之后
54
+ # (字典序,不包含marker)的bucket
55
+ # @return [Enumerator<Bucket>] Bucket的迭代器
56
+ def list_buckets(opts = {})
57
+ if @config.cname
58
+ fail ClientError, "Cannot list buckets for a CNAME endpoint."
59
+ end
60
+
61
+ Iterator::Buckets.new(@protocol, opts).to_enum
62
+ end
63
+
64
+ # 创建一个bucket
65
+ # @param name [String] Bucket名字
66
+ # @param opts [Hash] 创建Bucket的属性(可选)
67
+ # @option opts [:location] [String] 指定bucket所在的区域,默认为oss-cn-hangzhou
68
+ def create_bucket(name, opts = {})
69
+ @protocol.create_bucket(name, opts)
70
+ end
71
+
72
+ # 删除一个bucket
73
+ # @param name [String] Bucket名字
74
+ # @note 如果要删除的Bucket不为空(包含有object),则删除会失败
75
+ def delete_bucket(name)
76
+ @protocol.delete_bucket(name)
77
+ end
78
+
79
+ # 判断一个bucket是否存在
80
+ # @param name [String] Bucket名字
81
+ # @return [Boolean] 如果Bucket存在则返回true,否则返回false
82
+ def bucket_exists?(name)
83
+ exist = false
84
+
85
+ begin
86
+ @protocol.get_bucket_acl(name)
87
+ exist = true
88
+ rescue ServerError => e
89
+ raise unless e.http_code == 404
90
+ end
91
+
92
+ exist
93
+ end
94
+
95
+ alias :bucket_exist? :bucket_exists?
96
+
97
+ # 获取一个Bucket对象,用于操作bucket中的objects。
98
+ # @param name [String] Bucket名字
99
+ # @return [Bucket] Bucket对象
100
+ def get_bucket(name)
101
+ Bucket.new({:name => name}, @protocol)
102
+ end
103
+
104
+ end # Client
105
+ end # OSS
106
+ end # Aliyun
@@ -0,0 +1,39 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module AliyunSDK
4
+ module OSS
5
+
6
+ ##
7
+ # A place to store various configurations: credentials, api
8
+ # timeout, retry mechanism, etc
9
+ #
10
+ class Config < Common::Struct::Base
11
+
12
+ attrs :endpoint, :cname, :sts_token,
13
+ :access_key_id, :access_key_secret,
14
+ :open_timeout, :read_timeout
15
+
16
+ def initialize(opts = {})
17
+ super(opts)
18
+
19
+ @access_key_id = @access_key_id.strip if @access_key_id
20
+ @access_key_secret = @access_key_secret.strip if @access_key_secret
21
+ normalize_endpoint if endpoint
22
+ end
23
+
24
+ private
25
+
26
+ def normalize_endpoint
27
+ uri = URI.parse(endpoint)
28
+ uri = URI.parse("http://#{endpoint}") unless uri.scheme
29
+
30
+ if uri.scheme != 'http' and uri.scheme != 'https'
31
+ fail ClientError, "Only HTTP and HTTPS endpoint are accepted."
32
+ end
33
+
34
+ @endpoint = uri
35
+ end
36
+
37
+ end # Config
38
+ end # OSS
39
+ end # Aliyun
@@ -0,0 +1,255 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module AliyunSDK
4
+ module OSS
5
+ module Multipart
6
+ ##
7
+ # A multipart download transaction
8
+ #
9
+ class Download < Transaction
10
+
11
+ include Common::Logging
12
+
13
+ PART_SIZE = 10 * 1024 * 1024
14
+ READ_SIZE = 16 * 1024
15
+ NUM_THREAD = 10
16
+
17
+ def initialize(protocol, opts)
18
+ args = opts.dup
19
+ @protocol = protocol
20
+ @progress = args.delete(:progress)
21
+ @file = args.delete(:file)
22
+ @cpt_file = args.delete(:cpt_file)
23
+ super(args)
24
+
25
+ @object_meta = {}
26
+ @num_threads = options[:threads] || NUM_THREAD
27
+ @all_mutex = Mutex.new
28
+ @parts = []
29
+ @todo_mutex = Mutex.new
30
+ @todo_parts = []
31
+ end
32
+
33
+ # Run the download transaction, which includes 3 stages:
34
+ # * 1a. initiate(new downlaod) and divide parts
35
+ # * 1b. rebuild states(resumed download)
36
+ # * 2. download each unfinished part
37
+ # * 3. combine the downloaded parts into the final file
38
+ def run
39
+ logger.info("Begin download, file: #{@file}, "\
40
+ "checkpoint file: #{@cpt_file}, "\
41
+ "threads: #{@num_threads}")
42
+
43
+ # Rebuild transaction states from checkpoint file
44
+ # Or initiate new transaction states
45
+ rebuild
46
+
47
+ # Divide the target object into parts to download by ranges
48
+ divide_parts if @parts.empty?
49
+
50
+ # Download each part(object range)
51
+ @todo_parts = @parts.reject { |p| p[:done] }
52
+
53
+ (1..@num_threads).map {
54
+ Thread.new {
55
+ loop {
56
+ p = sync_get_todo_part
57
+ break unless p
58
+ download_part(p)
59
+ }
60
+ }
61
+ }.map(&:join)
62
+
63
+ # Combine the parts into the final file
64
+ commit
65
+
66
+ logger.info("Done download, file: #{@file}")
67
+ end
68
+
69
+ # Checkpoint structures:
70
+ # @example
71
+ # states = {
72
+ # :id => 'download_id',
73
+ # :file => 'file',
74
+ # :object_meta => {
75
+ # :etag => 'xxx',
76
+ # :size => 1024
77
+ # },
78
+ # :parts => [
79
+ # {:number => 1, :range => [0, 100], :md5 => 'xxx', :done => false},
80
+ # {:number => 2, :range => [100, 200], :md5 => 'yyy', :done => true}
81
+ # ],
82
+ # :md5 => 'states_md5'
83
+ # }
84
+ def checkpoint
85
+ logger.debug("Begin make checkpoint, disable_cpt: "\
86
+ "#{options[:disable_cpt] == true}")
87
+
88
+ ensure_object_not_changed
89
+
90
+ parts = sync_get_all_parts
91
+ states = {
92
+ :id => id,
93
+ :file => @file,
94
+ :object_meta => @object_meta,
95
+ :parts => parts
96
+ }
97
+
98
+ # report progress
99
+ if @progress
100
+ done = parts.count { |p| p[:done] }
101
+ @progress.call(done.to_f / parts.size) if done > 0
102
+ end
103
+
104
+ write_checkpoint(states, @cpt_file) unless options[:disable_cpt]
105
+
106
+ logger.debug("Done make checkpoint, states: #{states}")
107
+ end
108
+
109
+ private
110
+ # Combine the downloaded parts into the final file
111
+ # @todo avoid copy all part files
112
+ def commit
113
+ logger.info("Begin commit transaction, id: #{id}")
114
+
115
+ parts = sync_get_all_parts
116
+ # concat all part files into the target file
117
+ File.open(@file, 'w') do |w|
118
+ parts.sort{ |x, y| x[:number] <=> y[:number] }.each do |p|
119
+ File.open(get_part_file(p)) do |r|
120
+ w.write(r.read(READ_SIZE)) until r.eof?
121
+ end
122
+ end
123
+ end
124
+
125
+ File.delete(@cpt_file) unless options[:disable_cpt]
126
+ parts.each{ |p| File.delete(get_part_file(p)) }
127
+
128
+ logger.info("Done commit transaction, id: #{id}")
129
+ end
130
+
131
+ # Rebuild the states of the transaction from checkpoint file
132
+ def rebuild
133
+ logger.info("Begin rebuild transaction, checkpoint: #{@cpt_file}")
134
+
135
+ if options[:disable_cpt] || !File.exists?(@cpt_file)
136
+ initiate
137
+ else
138
+ states = load_checkpoint(@cpt_file)
139
+
140
+ states[:parts].select{ |p| p[:done] }.each do |p|
141
+ part_file = get_part_file(p)
142
+
143
+ unless File.exist?(part_file)
144
+ fail PartMissingError, "The part file is missing: #{part_file}."
145
+ end
146
+
147
+ if p[:md5] != get_file_md5(part_file)
148
+ fail PartInconsistentError,
149
+ "The part file is changed: #{part_file}."
150
+ end
151
+ end
152
+
153
+ @id = states[:id]
154
+ @object_meta = states[:object_meta]
155
+ @parts = states[:parts]
156
+ end
157
+
158
+ logger.info("Done rebuild transaction, states: #{states}")
159
+ end
160
+
161
+ def initiate
162
+ logger.info("Begin initiate transaction")
163
+
164
+ @id = generate_download_id
165
+ obj = @protocol.get_object_meta(bucket, object)
166
+ @object_meta = {
167
+ :etag => obj.etag,
168
+ :size => obj.size
169
+ }
170
+ checkpoint
171
+
172
+ logger.info("Done initiate transaction, id: #{id}")
173
+ end
174
+
175
+ # Download a part
176
+ def download_part(p)
177
+ logger.debug("Begin download part: #{p}")
178
+
179
+ part_file = get_part_file(p)
180
+ File.open(part_file, 'w') do |w|
181
+ @protocol.get_object(
182
+ bucket, object,
183
+ @options.merge(range: p[:range])) { |chunk| w.write(chunk) }
184
+ end
185
+
186
+ sync_update_part(p.merge(done: true, md5: get_file_md5(part_file)))
187
+
188
+ checkpoint
189
+
190
+ logger.debug("Done download part: #{p}")
191
+ end
192
+
193
+ # Devide the object to download into parts to download
194
+ def divide_parts
195
+ logger.info("Begin divide parts, object: #{@object}")
196
+
197
+ max_parts = 100
198
+ object_size = @object_meta[:size]
199
+ part_size =
200
+ [@options[:part_size] || PART_SIZE, object_size / max_parts].max
201
+ num_parts = (object_size - 1) / part_size + 1
202
+ @parts = (1..num_parts).map do |i|
203
+ {
204
+ :number => i,
205
+ :range => [(i - 1) * part_size, [i * part_size, object_size].min],
206
+ :done => false
207
+ }
208
+ end
209
+
210
+ checkpoint
211
+
212
+ logger.info("Done divide parts, parts: #{@parts}")
213
+ end
214
+
215
+ def sync_get_todo_part
216
+ @todo_mutex.synchronize {
217
+ @todo_parts.shift
218
+ }
219
+ end
220
+
221
+ def sync_update_part(p)
222
+ @all_mutex.synchronize {
223
+ @parts[p[:number] - 1] = p
224
+ }
225
+ end
226
+
227
+ def sync_get_all_parts
228
+ @all_mutex.synchronize {
229
+ @parts.dup
230
+ }
231
+ end
232
+
233
+ # Ensure file not changed during uploading
234
+ def ensure_object_not_changed
235
+ obj = @protocol.get_object_meta(bucket, object)
236
+ unless obj.etag == @object_meta[:etag]
237
+ fail ObjectInconsistentError,
238
+ "The object to download is changed: #{object}."
239
+ end
240
+ end
241
+
242
+ # Generate a download id
243
+ def generate_download_id
244
+ "download_#{bucket}_#{object}_#{Time.now.to_i}"
245
+ end
246
+
247
+ # Get name for part file
248
+ def get_part_file(p)
249
+ "#{@file}.part.#{p[:number]}"
250
+ end
251
+ end # Download
252
+
253
+ end # Multipart
254
+ end # OSS
255
+ end # Aliyun
@@ -0,0 +1,108 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require 'nokogiri'
4
+
5
+ module AliyunSDK
6
+ module OSS
7
+
8
+ ##
9
+ # ServerError represents exceptions from the OSS
10
+ # service. i.e. Client receives a HTTP response whose status is
11
+ # NOT OK. #message provides the error message and #to_s gives
12
+ # detailed information probably including the OSS request id.
13
+ #
14
+ class ServerError < Common::Exception
15
+
16
+ attr_reader :http_code, :error_code, :message, :request_id
17
+
18
+ def initialize(response)
19
+ @http_code = response.code
20
+ @attrs = {'RequestId' => get_request_id(response)}
21
+
22
+ doc = Nokogiri::XML(response.body) do |config|
23
+ config.options |= Nokogiri::XML::ParseOptions::NOBLANKS
24
+ end rescue nil
25
+
26
+ if doc and doc.root
27
+ doc.root.children.each do |n|
28
+ @attrs[n.name] = n.text
29
+ end
30
+ end
31
+
32
+ @error_code = @attrs['Code']
33
+ @message = @attrs['Message']
34
+ @request_id = @attrs['RequestId']
35
+ end
36
+
37
+ def message
38
+ msg = @attrs['Message'] || "UnknownError[#{http_code}]."
39
+ "#{msg} RequestId: #{request_id}"
40
+ end
41
+
42
+ def to_s
43
+ @attrs.merge({'HTTPCode' => @http_code}).map do |k, v|
44
+ [k, v].join(": ")
45
+ end.join(", ")
46
+ end
47
+
48
+ private
49
+
50
+ def get_request_id(response)
51
+ r = response.headers[:x_oss_request_id] if response.headers
52
+ r.to_s
53
+ end
54
+
55
+ end # ServerError
56
+
57
+ class CallbackError < ServerError
58
+ end # CallbackError
59
+
60
+ ##
61
+ # ClientError represents client exceptions caused mostly by
62
+ # invalid parameters.
63
+ #
64
+ class ClientError < Common::Exception
65
+ end # ClientError
66
+
67
+ ##
68
+ # FileInconsistentError happens in a resumable upload transaction,
69
+ # when the file to upload has changed during the uploading
70
+ # process. Which means the transaction cannot go on. Or user may
71
+ # have inconsistent data uploaded to OSS.
72
+ #
73
+ class FileInconsistentError < ClientError; end
74
+
75
+ ##
76
+ # ObjectInconsistentError happens in a resumable download transaction,
77
+ # when the object to download has changed during the downloading
78
+ # process. Which means the transaction cannot go on. Or user may
79
+ # have inconsistent data downloaded to OSS.
80
+ #
81
+ class ObjectInconsistentError < ClientError; end
82
+
83
+ ##
84
+ # PartMissingError happens in a resumable download transaction,
85
+ # when a downloaded part cannot be found as the client tries to
86
+ # resume download. The process cannot go on until the part is
87
+ # restored.
88
+ #
89
+ class PartMissingError < ClientError; end
90
+
91
+ ##
92
+ # PartMissingError happens in a resumable download transaction,
93
+ # when a downloaded part has changed(MD5 mismatch) as the client
94
+ # tries to resume download. The process cannot go on until the
95
+ # part is restored.
96
+ #
97
+ class PartInconsistentError < ClientError; end
98
+
99
+ ##
100
+ # CheckpointBrokenError happens in a resumable upload/download
101
+ # transaction, when the client finds the checkpoint file has
102
+ # changed(MD5 mismatch) as it tries to resume upload/download. The
103
+ # process cannot go on until the checkpoint file is restored.
104
+ #
105
+ class CheckpointBrokenError < ClientError; end
106
+
107
+ end # OSS
108
+ end # Aliyun