aliyun-oss-ruby-sdk 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +95 -0
  3. data/README.md +423 -0
  4. data/examples/aliyun/oss/bucket.rb +144 -0
  5. data/examples/aliyun/oss/callback.rb +61 -0
  6. data/examples/aliyun/oss/object.rb +182 -0
  7. data/examples/aliyun/oss/resumable_download.rb +42 -0
  8. data/examples/aliyun/oss/resumable_upload.rb +49 -0
  9. data/examples/aliyun/oss/streaming.rb +124 -0
  10. data/examples/aliyun/oss/using_sts.rb +48 -0
  11. data/examples/aliyun/sts/assume_role.rb +59 -0
  12. data/lib/aliyun_sdk/common.rb +6 -0
  13. data/lib/aliyun_sdk/common/exception.rb +18 -0
  14. data/lib/aliyun_sdk/common/logging.rb +46 -0
  15. data/lib/aliyun_sdk/common/struct.rb +56 -0
  16. data/lib/aliyun_sdk/oss.rb +16 -0
  17. data/lib/aliyun_sdk/oss/bucket.rb +661 -0
  18. data/lib/aliyun_sdk/oss/client.rb +106 -0
  19. data/lib/aliyun_sdk/oss/config.rb +39 -0
  20. data/lib/aliyun_sdk/oss/download.rb +255 -0
  21. data/lib/aliyun_sdk/oss/exception.rb +108 -0
  22. data/lib/aliyun_sdk/oss/http.rb +338 -0
  23. data/lib/aliyun_sdk/oss/iterator.rb +92 -0
  24. data/lib/aliyun_sdk/oss/multipart.rb +74 -0
  25. data/lib/aliyun_sdk/oss/object.rb +15 -0
  26. data/lib/aliyun_sdk/oss/protocol.rb +1499 -0
  27. data/lib/aliyun_sdk/oss/struct.rb +208 -0
  28. data/lib/aliyun_sdk/oss/upload.rb +238 -0
  29. data/lib/aliyun_sdk/oss/util.rb +89 -0
  30. data/lib/aliyun_sdk/sts.rb +9 -0
  31. data/lib/aliyun_sdk/sts/client.rb +38 -0
  32. data/lib/aliyun_sdk/sts/config.rb +22 -0
  33. data/lib/aliyun_sdk/sts/exception.rb +53 -0
  34. data/lib/aliyun_sdk/sts/protocol.rb +130 -0
  35. data/lib/aliyun_sdk/sts/struct.rb +64 -0
  36. data/lib/aliyun_sdk/sts/util.rb +48 -0
  37. data/lib/aliyun_sdk/version.rb +7 -0
  38. data/spec/aliyun/oss/bucket_spec.rb +597 -0
  39. data/spec/aliyun/oss/client/bucket_spec.rb +554 -0
  40. data/spec/aliyun/oss/client/client_spec.rb +297 -0
  41. data/spec/aliyun/oss/client/resumable_download_spec.rb +220 -0
  42. data/spec/aliyun/oss/client/resumable_upload_spec.rb +413 -0
  43. data/spec/aliyun/oss/http_spec.rb +83 -0
  44. data/spec/aliyun/oss/multipart_spec.rb +686 -0
  45. data/spec/aliyun/oss/object_spec.rb +785 -0
  46. data/spec/aliyun/oss/service_spec.rb +142 -0
  47. data/spec/aliyun/oss/util_spec.rb +50 -0
  48. data/spec/aliyun/sts/client_spec.rb +150 -0
  49. data/spec/aliyun/sts/util_spec.rb +39 -0
  50. data/tests/config.rb +31 -0
  51. data/tests/test_content_encoding.rb +54 -0
  52. data/tests/test_content_type.rb +95 -0
  53. data/tests/test_custom_headers.rb +70 -0
  54. data/tests/test_encoding.rb +77 -0
  55. data/tests/test_large_file.rb +66 -0
  56. data/tests/test_multipart.rb +97 -0
  57. data/tests/test_object_acl.rb +49 -0
  58. data/tests/test_object_key.rb +68 -0
  59. data/tests/test_object_url.rb +69 -0
  60. data/tests/test_resumable.rb +40 -0
  61. metadata +240 -0
@@ -0,0 +1,208 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require 'base64'
4
+ require 'json'
5
+ require 'uri'
6
+
7
+ module AliyunSDK
8
+ module OSS
9
+
10
+ ##
11
+ # Access Control List, it controls how the bucket/object can be
12
+ # accessed.
13
+ # * public-read-write: allow access(read&write) anonymously
14
+ # * public-read: allow read anonymously
15
+ # * private: access must be signatured
16
+ #
17
+ module ACL
18
+ PUBLIC_READ_WRITE = "public-read-write"
19
+ PUBLIC_READ = "public-read"
20
+ PRIVATE = "private"
21
+ end # ACL
22
+
23
+ ##
24
+ # A OSS object may carry some metas(String key-value pairs) with
25
+ # it. MetaDirective specifies what to do with the metas in the
26
+ # copy process.
27
+ # * COPY: metas are copied from the source object to the dest
28
+ # object
29
+ # * REPLACE: source object's metas are NOT copied, use user
30
+ # provided metas for the dest object
31
+ #
32
+ module MetaDirective
33
+ COPY = "COPY"
34
+ REPLACE = "REPLACE"
35
+ end # MetaDirective
36
+
37
+ ##
38
+ # The object key may contains unicode charactors which cannot be
39
+ # encoded in the request/response body(XML). KeyEncoding specifies
40
+ # the encoding type for the object key.
41
+ # * url: the object key is url-encoded
42
+ # @note url-encoding is the only supported KeyEncoding type
43
+ #
44
+ module KeyEncoding
45
+ URL = "url"
46
+
47
+ @@all = [URL]
48
+
49
+ def self.include?(enc)
50
+ all.include?(enc)
51
+ end
52
+
53
+ def self.all
54
+ @@all
55
+ end
56
+ end # KeyEncoding
57
+
58
+ ##
59
+ # Bucket Logging setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/logging.html OSS Bucket logging}
60
+ # Attributes:
61
+ # * enable [Boolean] whether to enable bucket logging
62
+ # * target_bucket [String] the target bucket to store access logs
63
+ # * target_prefix [String] the target object prefix to store access logs
64
+ # @example Enable bucket logging
65
+ # bucket.logging = BucketLogging.new(
66
+ # :enable => true, :target_bucket => 'log_bucket', :target_prefix => 'my-log')
67
+ # @example Disable bucket logging
68
+ # bucket.logging = BucketLogging.new(:enable => false)
69
+ class BucketLogging < Common::Struct::Base
70
+ attrs :enable, :target_bucket, :target_prefix
71
+
72
+ def enabled?
73
+ enable == true
74
+ end
75
+ end
76
+
77
+ ##
78
+ # Bucket website setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/host-static-website.html OSS Website hosting}
79
+ # Attributes:
80
+ # * enable [Boolean] whether to enable website hosting for the bucket
81
+ # * index [String] the index object as the index page for the website
82
+ # * error [String] the error object as the error page for the website
83
+ class BucketWebsite < Common::Struct::Base
84
+ attrs :enable, :index, :error
85
+
86
+ def enabled?
87
+ enable == true
88
+ end
89
+ end
90
+
91
+ ##
92
+ # Bucket referer setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/referer-white-list.html OSS Website hosting}
93
+ # Attributes:
94
+ # * allow_empty [Boolean] whether to allow requests with empty "Referer"
95
+ # * whitelist [Array<String>] the allowed origins for requests
96
+ class BucketReferer < Common::Struct::Base
97
+ attrs :allow_empty, :whitelist
98
+
99
+ def allow_empty?
100
+ allow_empty == true
101
+ end
102
+ end
103
+
104
+ ##
105
+ # LifeCycle rule for bucket. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/lifecycle.html OSS Bucket LifeCycle}
106
+ # Attributes:
107
+ # * id [String] the unique id of a rule
108
+ # * enabled [Boolean] whether to enable this rule
109
+ # * prefix [String] the prefix objects to apply this rule
110
+ # * expiry [Date] or [Fixnum] the expire time of objects
111
+ # * if expiry is a Date, it specifies the absolute date to
112
+ # expire objects
113
+ # * if expiry is a Fixnum, it specifies the relative date to
114
+ # expire objects: how many days after the object's last
115
+ # modification time to expire the object
116
+ # @example Specify expiry as Date
117
+ # LifeCycleRule.new(
118
+ # :id => 'rule1',
119
+ # :enabled => true,
120
+ # :prefix => 'foo/',
121
+ # :expiry => Date.new(2016, 1, 1))
122
+ # @example Specify expiry as days
123
+ # LifeCycleRule.new(
124
+ # :id => 'rule1',
125
+ # :enabled => true,
126
+ # :prefix => 'foo/',
127
+ # :expiry => 15)
128
+ # @note the expiry date is treated as UTC time
129
+ class LifeCycleRule < Common::Struct::Base
130
+
131
+ attrs :id, :enable, :prefix, :expiry
132
+
133
+ def enabled?
134
+ enable == true
135
+ end
136
+ end # LifeCycleRule
137
+
138
+ ##
139
+ # CORS rule for bucket. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/referer-white-list.html OSS CORS}
140
+ # Attributes:
141
+ # * allowed_origins [Array<String>] the allowed origins
142
+ # * allowed_methods [Array<String>] the allowed methods
143
+ # * allowed_headers [Array<String>] the allowed headers
144
+ # * expose_headers [Array<String>] the expose headers
145
+ # * max_age_seconds [Integer] the max age seconds
146
+ class CORSRule < Common::Struct::Base
147
+
148
+ attrs :allowed_origins, :allowed_methods, :allowed_headers,
149
+ :expose_headers, :max_age_seconds
150
+
151
+ end # CORSRule
152
+
153
+ ##
154
+ # Callback represents a HTTP call made by OSS to user's
155
+ # application server after an event happens, such as an object is
156
+ # successfully uploaded to OSS. See: {https://help.aliyun.com/document_detail/oss/api-reference/object/Callback.html}
157
+ # Attributes:
158
+ # * url [String] the URL *WITHOUT* the query string
159
+ # * query [Hash] the query to generate query string
160
+ # * body [String] the body of the request
161
+ # * content_type [String] the Content-Type of the request
162
+ # * host [String] the Host in HTTP header for this request
163
+ class Callback < Common::Struct::Base
164
+
165
+ attrs :url, :query, :body, :content_type, :host
166
+
167
+ include Common::Logging
168
+
169
+ def serialize
170
+ query_string = (query || {}).map { |k, v|
171
+ [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=') }.join('&')
172
+
173
+ cb = {
174
+ 'callbackUrl' => "#{normalize_url(url)}?#{query_string}",
175
+ 'callbackBody' => body,
176
+ 'callbackBodyType' => content_type || default_content_type
177
+ }
178
+ cb['callbackHost'] = host if host
179
+
180
+ logger.debug("Callback json: #{cb}")
181
+
182
+ Base64.strict_encode64(cb.to_json)
183
+ end
184
+
185
+ private
186
+ def normalize_url(url)
187
+ uri = URI.parse(url)
188
+ uri = URI.parse("http://#{url}") unless uri.scheme
189
+
190
+ if uri.scheme != 'http' and uri.scheme != 'https'
191
+ fail ClientError, "Only HTTP and HTTPS endpoint are accepted."
192
+ end
193
+
194
+ unless uri.query.nil?
195
+ fail ClientError, "Query parameters should not appear in URL."
196
+ end
197
+
198
+ uri.to_s
199
+ end
200
+
201
+ def default_content_type
202
+ "application/x-www-form-urlencoded"
203
+ end
204
+
205
+ end # Callback
206
+
207
+ end # OSS
208
+ end # Aliyun
@@ -0,0 +1,238 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module AliyunSDK
4
+ module OSS
5
+ module Multipart
6
+ ##
7
+ # A multipart upload transaction
8
+ #
9
+ class Upload < Transaction
10
+
11
+ include Common::Logging
12
+
13
+ PART_SIZE = 10 * 1024 * 1024
14
+ READ_SIZE = 16 * 1024
15
+ NUM_THREAD = 10
16
+
17
+ def initialize(protocol, opts)
18
+ args = opts.dup
19
+ @protocol = protocol
20
+ @progress = args.delete(:progress)
21
+ @file = args.delete(:file)
22
+ @cpt_file = args.delete(:cpt_file)
23
+ super(args)
24
+
25
+ @file_meta = {}
26
+ @num_threads = options[:threads] || NUM_THREAD
27
+ @all_mutex = Mutex.new
28
+ @parts = []
29
+ @todo_mutex = Mutex.new
30
+ @todo_parts = []
31
+ end
32
+
33
+ # Run the upload transaction, which includes 3 stages:
34
+ # * 1a. initiate(new upload) and divide parts
35
+ # * 1b. rebuild states(resumed upload)
36
+ # * 2. upload each unfinished part
37
+ # * 3. commit the multipart upload transaction
38
+ def run
39
+ logger.info("Begin upload, file: #{@file}, "\
40
+ "checkpoint file: #{@cpt_file}, "\
41
+ "threads: #{@num_threads}")
42
+
43
+ # Rebuild transaction states from checkpoint file
44
+ # Or initiate new transaction states
45
+ rebuild
46
+
47
+ # Divide the file to upload into parts to upload separately
48
+ divide_parts if @parts.empty?
49
+
50
+ # Upload each part
51
+ @todo_parts = @parts.reject { |p| p[:done] }
52
+
53
+ (1..@num_threads).map {
54
+ Thread.new {
55
+ loop {
56
+ p = sync_get_todo_part
57
+ break unless p
58
+ upload_part(p)
59
+ }
60
+ }
61
+ }.map(&:join)
62
+
63
+ # Commit the multipart upload transaction
64
+ commit
65
+
66
+ logger.info("Done upload, file: #{@file}")
67
+ end
68
+
69
+ # Checkpoint structures:
70
+ # @example
71
+ # states = {
72
+ # :id => 'upload_id',
73
+ # :file => 'file',
74
+ # :file_meta => {
75
+ # :mtime => Time.now,
76
+ # :md5 => 1024
77
+ # },
78
+ # :parts => [
79
+ # {:number => 1, :range => [0, 100], :done => false},
80
+ # {:number => 2, :range => [100, 200], :done => true}
81
+ # ],
82
+ # :md5 => 'states_md5'
83
+ # }
84
+ def checkpoint
85
+ logger.debug("Begin make checkpoint, disable_cpt: "\
86
+ "#{options[:disable_cpt] == true}")
87
+
88
+ ensure_file_not_changed
89
+
90
+ parts = sync_get_all_parts
91
+ states = {
92
+ :id => id,
93
+ :file => @file,
94
+ :file_meta => @file_meta,
95
+ :parts => parts
96
+ }
97
+
98
+ # report progress
99
+ if @progress
100
+ done = parts.count { |p| p[:done] }
101
+ @progress.call(done.to_f / parts.size) if done > 0
102
+ end
103
+
104
+ write_checkpoint(states, @cpt_file) unless options[:disable_cpt]
105
+
106
+ logger.debug("Done make checkpoint, states: #{states}")
107
+ end
108
+
109
+ private
110
+ # Commit the transaction when all parts are succefully uploaded
111
+ # @todo handle undefined behaviors: commit succeeds in server
112
+ # but return error in client
113
+ def commit
114
+ logger.info("Begin commit transaction, id: #{id}")
115
+
116
+ parts = sync_get_all_parts.map{ |p|
117
+ Part.new(:number => p[:number], :etag => p[:etag])
118
+ }
119
+ @protocol.complete_multipart_upload(
120
+ bucket, object, id, parts, @options[:callback])
121
+
122
+ File.delete(@cpt_file) unless options[:disable_cpt]
123
+
124
+ logger.info("Done commit transaction, id: #{id}")
125
+ end
126
+
127
+ # Rebuild the states of the transaction from checkpoint file
128
+ def rebuild
129
+ logger.info("Begin rebuild transaction, checkpoint: #{@cpt_file}")
130
+
131
+ if options[:disable_cpt] || !File.exists?(@cpt_file)
132
+ initiate
133
+ else
134
+ states = load_checkpoint(@cpt_file)
135
+
136
+ if states[:file_md5] != @file_meta[:md5]
137
+ fail FileInconsistentError.new("The file to upload is changed.")
138
+ end
139
+
140
+ @id = states[:id]
141
+ @file_meta = states[:file_meta]
142
+ @parts = states[:parts]
143
+ end
144
+
145
+ logger.info("Done rebuild transaction, states: #{states}")
146
+ end
147
+
148
+ def initiate
149
+ logger.info("Begin initiate transaction")
150
+
151
+ @id = @protocol.initiate_multipart_upload(bucket, object, options)
152
+ @file_meta = {
153
+ :mtime => File.mtime(@file),
154
+ :md5 => get_file_md5(@file)
155
+ }
156
+ checkpoint
157
+
158
+ logger.info("Done initiate transaction, id: #{id}")
159
+ end
160
+
161
+ # Upload a part
162
+ def upload_part(p)
163
+ logger.debug("Begin upload part: #{p}")
164
+
165
+ result = nil
166
+ File.open(@file) do |f|
167
+ range = p[:range]
168
+ pos = range.first
169
+ f.seek(pos)
170
+
171
+ result = @protocol.upload_part(bucket, object, id, p[:number]) do |sw|
172
+ while pos < range.at(1)
173
+ bytes = [READ_SIZE, range.at(1) - pos].min
174
+ sw << f.read(bytes)
175
+ pos += bytes
176
+ end
177
+ end
178
+ end
179
+
180
+ sync_update_part(p.merge(done: true, etag: result.etag))
181
+
182
+ checkpoint
183
+
184
+ logger.debug("Done upload part: #{p}")
185
+ end
186
+
187
+ # Devide the file into parts to upload
188
+ def divide_parts
189
+ logger.info("Begin divide parts, file: #{@file}")
190
+
191
+ max_parts = 10000
192
+ file_size = File.size(@file)
193
+ part_size = [@options[:part_size] || PART_SIZE, file_size / max_parts].max
194
+ num_parts = (file_size - 1) / part_size + 1
195
+ @parts = (1..num_parts).map do |i|
196
+ {
197
+ :number => i,
198
+ :range => [(i-1) * part_size, [i * part_size, file_size].min],
199
+ :done => false
200
+ }
201
+ end
202
+
203
+ checkpoint
204
+
205
+ logger.info("Done divide parts, parts: #{@parts}")
206
+ end
207
+
208
+ def sync_get_todo_part
209
+ @todo_mutex.synchronize {
210
+ @todo_parts.shift
211
+ }
212
+ end
213
+
214
+ def sync_update_part(p)
215
+ @all_mutex.synchronize {
216
+ @parts[p[:number] - 1] = p
217
+ }
218
+ end
219
+
220
+ def sync_get_all_parts
221
+ @all_mutex.synchronize {
222
+ @parts.dup
223
+ }
224
+ end
225
+
226
+ # Ensure file not changed during uploading
227
+ def ensure_file_not_changed
228
+ return if File.mtime(@file) == @file_meta[:mtime]
229
+
230
+ if @file_meta[:md5] != get_file_md5(@file)
231
+ fail FileInconsistentError, "The file to upload is changed."
232
+ end
233
+ end
234
+ end # Upload
235
+
236
+ end # Multipart
237
+ end # OSS
238
+ end # Aliyun
@@ -0,0 +1,89 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require 'time'
4
+ require 'base64'
5
+ require 'openssl'
6
+ require 'digest/md5'
7
+
8
+ module AliyunSDK
9
+ module OSS
10
+ ##
11
+ # Util functions to help generate formatted Date, signatures,
12
+ # etc.
13
+ #
14
+ module Util
15
+
16
+ # Prefix for OSS specific HTTP headers
17
+ HEADER_PREFIX = "x-oss-"
18
+
19
+ class << self
20
+
21
+ include Common::Logging
22
+
23
+ # Calculate request signatures
24
+ def get_signature(key, verb, headers, resources)
25
+ logger.debug("Sign, headers: #{headers}, resources: #{resources}")
26
+
27
+ content_md5 = headers['content-md5'] || ""
28
+ content_type = headers['content-type'] || ""
29
+ date = headers['date']
30
+
31
+ cano_headers = headers.select { |k, v| k.start_with?(HEADER_PREFIX) }
32
+ .map { |k, v| [k.downcase.strip, v.strip] }
33
+ .sort.map { |k, v| [k, v].join(":") + "\n" }.join
34
+
35
+ cano_res = resources[:path] || "/"
36
+ sub_res = (resources[:sub_res] || {})
37
+ .sort.map { |k, v| v ? [k, v].join("=") : k }.join("&")
38
+ cano_res += "?#{sub_res}" unless sub_res.empty?
39
+
40
+ string_to_sign =
41
+ "#{verb}\n#{content_md5}\n#{content_type}\n#{date}\n" +
42
+ "#{cano_headers}#{cano_res}"
43
+
44
+ Util.sign(key, string_to_sign)
45
+ end
46
+
47
+ # Sign a string using HMAC and BASE64
48
+ # @param [String] key the secret key
49
+ # @param [String] string_to_sign the string to sign
50
+ # @return [String] the signature
51
+ def sign(key, string_to_sign)
52
+ logger.debug("String to sign: #{string_to_sign}")
53
+
54
+ Base64.strict_encode64(
55
+ OpenSSL::HMAC.digest('sha1', key, string_to_sign))
56
+ end
57
+
58
+ # Calculate content md5
59
+ def get_content_md5(content)
60
+ Base64.strict_encode64(OpenSSL::Digest::MD5.digest(content))
61
+ end
62
+
63
+ # Symbolize keys in Hash, recursively
64
+ def symbolize(v)
65
+ if v.is_a?(Hash)
66
+ result = {}
67
+ v.each_key { |k| result[k.to_sym] = symbolize(v[k]) }
68
+ result
69
+ elsif v.is_a?(Array)
70
+ result = []
71
+ v.each { |x| result << symbolize(x) }
72
+ result
73
+ else
74
+ v
75
+ end
76
+ end
77
+
78
+ end # self
79
+ end # Util
80
+ end # OSS
81
+ end # Aliyun
82
+
83
+ # Monkey patch to support #to_bool
84
+ class String
85
+ def to_bool
86
+ return true if self =~ /^true$/i
87
+ false
88
+ end
89
+ end