qiniu-rs 3.2.1 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/{docs/NEWS.md → CHANGELOG.md} +7 -1
- data/Gemfile.lock +10 -10
- data/lib/qiniu/rs/exceptions.rb +0 -3
- data/lib/qiniu/rs/up.rb +60 -71
- data/lib/qiniu/rs/version.rb +1 -1
- data/qiniu-rs.gemspec +1 -1
- data/spec/qiniu/rs/up_spec.rb +1 -3
- data/spec/spec_helper.rb +2 -8
- metadata +7 -7
@@ -1,5 +1,11 @@
|
|
1
1
|
## CHANGE LOG
|
2
2
|
|
3
|
+
### v3.2.2
|
4
|
+
|
5
|
+
fixed E701 error
|
6
|
+
|
7
|
+
断点续上传根据 mkblk 返回的 host 字段进行 bput 和 mkfile ,规避由于DNS智能解析造成的分布式并行块上传会出现上下文不连贯导致的 E701 问题。
|
8
|
+
|
3
9
|
### v3.2.1
|
4
10
|
|
5
11
|
allow images uploaded auto-orient.
|
@@ -22,4 +28,4 @@ allow files uploaded auto callback some APIs (like imageInfo, exif, etc…), and
|
|
22
28
|
参考:
|
23
29
|
|
24
30
|
1. [[API] 生成上传授权凭证 uploadToken 之 escape 参数详解](http://docs.qiniutek.com/v3/api/io/#escape-expression)
|
25
|
-
2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
|
31
|
+
2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
qiniu-rs (3.2.
|
4
|
+
qiniu-rs (3.2.2)
|
5
5
|
json (~> 1.7)
|
6
6
|
mime-types (~> 1.19)
|
7
7
|
rest-client (~> 1.6)
|
@@ -14,17 +14,17 @@ GEM
|
|
14
14
|
fakeweb (1.3.0)
|
15
15
|
json (1.7.5)
|
16
16
|
mime-types (1.19)
|
17
|
-
rake (0.
|
17
|
+
rake (10.0.2)
|
18
18
|
rest-client (1.6.7)
|
19
19
|
mime-types (>= 1.16)
|
20
|
-
rspec (2.
|
21
|
-
rspec-core (~> 2.
|
22
|
-
rspec-expectations (~> 2.
|
23
|
-
rspec-mocks (~> 2.
|
24
|
-
rspec-core (2.
|
25
|
-
rspec-expectations (2.
|
20
|
+
rspec (2.12.0)
|
21
|
+
rspec-core (~> 2.12.0)
|
22
|
+
rspec-expectations (~> 2.12.0)
|
23
|
+
rspec-mocks (~> 2.12.0)
|
24
|
+
rspec-core (2.12.1)
|
25
|
+
rspec-expectations (2.12.0)
|
26
26
|
diff-lcs (~> 1.1.3)
|
27
|
-
rspec-mocks (2.
|
27
|
+
rspec-mocks (2.12.0)
|
28
28
|
ruby-hmac (0.4.0)
|
29
29
|
|
30
30
|
PLATFORMS
|
@@ -33,5 +33,5 @@ PLATFORMS
|
|
33
33
|
DEPENDENCIES
|
34
34
|
fakeweb (~> 1.3)
|
35
35
|
qiniu-rs!
|
36
|
-
rake (
|
36
|
+
rake (>= 0.9)
|
37
37
|
rspec (~> 2.11)
|
data/lib/qiniu/rs/exceptions.rb
CHANGED
data/lib/qiniu/rs/up.rb
CHANGED
@@ -14,6 +14,9 @@ module Qiniu
|
|
14
14
|
module RS
|
15
15
|
module UP
|
16
16
|
|
17
|
+
PROGRESS_TMP_FILE = 'progresses'
|
18
|
+
CHECKSUM_TMP_FILE = 'ctxes'
|
19
|
+
|
17
20
|
module AbstractClass
|
18
21
|
class ChunkProgressNotifier
|
19
22
|
include Qiniu::RS::Abstract
|
@@ -29,78 +32,58 @@ module Qiniu
|
|
29
32
|
end
|
30
33
|
|
31
34
|
class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier
|
35
|
+
attr_reader :tmpdata
|
32
36
|
def initialize(id)
|
33
|
-
@
|
37
|
+
@tmpdata = UP::TmpData.new(id, PROGRESS_TMP_FILE)
|
34
38
|
end
|
35
39
|
def notify(index, progress)
|
36
|
-
@
|
37
|
-
logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" +
|
38
|
-
"{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
|
40
|
+
@tmpdata.set(index, progress)
|
41
|
+
logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s
|
39
42
|
Utils.debug(logmsg)
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
43
46
|
class BlockProgressNotifier < AbstractClass::BlockProgressNotifier
|
47
|
+
attr_reader :tmpdata
|
44
48
|
def initialize(id)
|
45
|
-
@
|
49
|
+
@tmpdata = UP::TmpData.new(id, CHECKSUM_TMP_FILE)
|
46
50
|
end
|
47
51
|
def notify(index, checksum)
|
48
|
-
@
|
52
|
+
@tmpdata.set(index, checksum)
|
53
|
+
Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded."
|
49
54
|
Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded."
|
50
55
|
end
|
51
56
|
end
|
52
57
|
|
53
|
-
class
|
54
|
-
def initialize(
|
55
|
-
@
|
56
|
-
@tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
|
58
|
+
class TmpData
|
59
|
+
def initialize(dir, filename)
|
60
|
+
@tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + dir
|
57
61
|
FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir)
|
58
|
-
@
|
59
|
-
@progress_file = @tmpdir + File::SEPARATOR + 'progresses'
|
60
|
-
end
|
61
|
-
|
62
|
-
def get_checksums
|
63
|
-
File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
|
64
|
-
end
|
65
|
-
|
66
|
-
def get_progresses
|
67
|
-
File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
|
62
|
+
@tmpfile = @tmpdir + File::SEPARATOR + filename
|
68
63
|
end
|
69
64
|
|
70
|
-
def
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
YAML::dump(checksums, f)
|
75
|
-
Utils.debug %Q(Updating tmpfile: #{@checksum_file})
|
65
|
+
def init(values)
|
66
|
+
File.open(@tmpfile, "w") do |f|
|
67
|
+
YAML::dump(values, f)
|
68
|
+
Utils.debug %Q(Initializing tmpfile: #{@tmpfile})
|
76
69
|
end
|
77
70
|
end
|
78
71
|
|
79
|
-
def
|
80
|
-
|
81
|
-
progresses[index] = progress
|
82
|
-
File.open(@progress_file, "w") do |f|
|
83
|
-
YAML::dump(progresses, f)
|
84
|
-
Utils.debug %Q(Updating tmpfile: #{@progress_file})
|
85
|
-
end
|
72
|
+
def all
|
73
|
+
File.exist?(@tmpfile) ? YAML.load_file(@tmpfile) : []
|
86
74
|
end
|
87
75
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
def init_progresses(progresses)
|
96
|
-
File.open(@progress_file, "w") do |f|
|
97
|
-
YAML::dump(progresses, f)
|
98
|
-
Utils.debug %Q(Initializing tmpfile: #{@progress_file})
|
76
|
+
def set(index, value)
|
77
|
+
values = all
|
78
|
+
values[index] = value
|
79
|
+
File.open(@tmpfile, "w") do |f|
|
80
|
+
YAML::dump(values, f)
|
81
|
+
Utils.debug %Q(Updating tmpfile: #{@tmpfile})
|
99
82
|
end
|
100
83
|
end
|
101
84
|
|
102
85
|
def sweep!
|
103
|
-
FileUtils.rm_r(@tmpdir)
|
86
|
+
FileUtils.rm_r(@tmpdir) if Dir.exists?(@tmpdir)
|
104
87
|
end
|
105
88
|
end
|
106
89
|
|
@@ -162,20 +145,21 @@ module Qiniu
|
|
162
145
|
end
|
163
146
|
|
164
147
|
def _new_block_put_progress_data
|
165
|
-
{:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
|
148
|
+
{:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
|
166
149
|
end
|
167
150
|
|
168
|
-
def _call_binary_with_token(uptoken, url, data, retry_times = 0)
|
151
|
+
def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
|
169
152
|
options = {
|
170
153
|
:method => :post,
|
171
154
|
:content_type => 'application/octet-stream',
|
172
155
|
:upload_signature_token => uptoken
|
173
156
|
}
|
157
|
+
options[:content_type] = content_type if !content_type.nil? && !content_type.empty?
|
174
158
|
code, data = http_request url, data, options
|
175
159
|
unless Utils.is_response_ok?(code)
|
176
160
|
retry_times += 1
|
177
161
|
if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
|
178
|
-
return _call_binary_with_token(uptoken, url, data, retry_times)
|
162
|
+
return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
|
179
163
|
end
|
180
164
|
end
|
181
165
|
[code, data]
|
@@ -186,8 +170,8 @@ module Qiniu
|
|
186
170
|
_call_binary_with_token(uptoken, url, body)
|
187
171
|
end
|
188
172
|
|
189
|
-
def _putblock(uptoken, ctx, offset, body)
|
190
|
-
url =
|
173
|
+
def _putblock(uphost, uptoken, ctx, offset, body)
|
174
|
+
url = uphost + "/bput/#{ctx}/#{offset}"
|
191
175
|
_call_binary_with_token(uptoken, url, body)
|
192
176
|
end
|
193
177
|
|
@@ -214,6 +198,7 @@ module Qiniu
|
|
214
198
|
progress[:offset] = body_length
|
215
199
|
progress[:restsize] = block_size - body_length
|
216
200
|
progress[:status_code] = code
|
201
|
+
progress[:host] = data["host"]
|
217
202
|
if !notifier.nil? && notifier.respond_to?("notify")
|
218
203
|
notifier.notify(block_index, progress)
|
219
204
|
end
|
@@ -236,13 +221,14 @@ module Qiniu
|
|
236
221
|
if result_length != body_length
|
237
222
|
raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
|
238
223
|
end
|
239
|
-
code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
|
224
|
+
code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
|
240
225
|
body_crc32 = Zlib.crc32(body)
|
241
226
|
if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
|
242
227
|
progress[:ctx] = data["ctx"]
|
243
228
|
progress[:offset] += body_length
|
244
229
|
progress[:restsize] -= body_length
|
245
230
|
progress[:status_code] = code
|
231
|
+
progress[:host] = data["host"]
|
246
232
|
if !notifier.nil? && notifier.respond_to?("notify")
|
247
233
|
notifier.notify(block_index, progress)
|
248
234
|
end
|
@@ -280,7 +266,8 @@ module Qiniu
|
|
280
266
|
end
|
281
267
|
code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
|
282
268
|
if Utils.is_response_ok?(code)
|
283
|
-
checksums[block_index] = data["checksum"]
|
269
|
+
#checksums[block_index] = data["checksum"]
|
270
|
+
checksums[block_index] = data["ctx"]
|
284
271
|
if !block_notifier.nil? && block_notifier.respond_to?("notify")
|
285
272
|
block_notifier.notify(block_index, checksums[block_index])
|
286
273
|
end
|
@@ -290,7 +277,7 @@ module Qiniu
|
|
290
277
|
return [code, data]
|
291
278
|
end
|
292
279
|
|
293
|
-
def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
|
280
|
+
def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
|
294
281
|
path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
|
295
282
|
path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
|
296
283
|
path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
|
@@ -298,37 +285,39 @@ module Qiniu
|
|
298
285
|
callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
|
299
286
|
path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
|
300
287
|
path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
|
301
|
-
url =
|
302
|
-
body = ''
|
303
|
-
checksums.each do |checksum|
|
304
|
-
|
305
|
-
end
|
306
|
-
|
288
|
+
url = uphost + path
|
289
|
+
#body = ''
|
290
|
+
#checksums.each do |checksum|
|
291
|
+
# body += Utils.urlsafe_base64_decode(checksum)
|
292
|
+
#end
|
293
|
+
body = checksums.join(',')
|
294
|
+
_call_binary_with_token(uptoken, url, body, 'text/plain')
|
307
295
|
end
|
308
296
|
|
309
297
|
def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
|
310
298
|
block_count = _block_count(fsize)
|
311
|
-
|
312
|
-
|
313
|
-
progresses =
|
314
|
-
if checksums.empty?
|
315
|
-
block_count.times{checksums << ''}
|
316
|
-
progress_data.init_checksums(checksums)
|
317
|
-
end
|
299
|
+
chunk_notifier = ChunkProgressNotifier.new(key)
|
300
|
+
block_notifier = BlockProgressNotifier.new(key)
|
301
|
+
progresses = chunk_notifier.tmpdata.all
|
318
302
|
if progresses.empty?
|
319
303
|
block_count.times{progresses << _new_block_put_progress_data}
|
320
|
-
|
304
|
+
chunk_notifier.tmpdata.init(progresses)
|
305
|
+
end
|
306
|
+
checksums = block_notifier.tmpdata.all
|
307
|
+
if checksums.empty?
|
308
|
+
block_count.times{checksums << ''}
|
309
|
+
block_notifier.tmpdata.init(checksums)
|
321
310
|
end
|
322
|
-
chunk_notifier = ChunkProgressNotifier.new(key)
|
323
|
-
block_notifier = BlockProgressNotifier.new(key)
|
324
311
|
code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
|
325
312
|
if Utils.is_response_ok?(code)
|
313
|
+
uphost = data["host"]
|
326
314
|
entry_uri = bucket + ':' + key
|
327
|
-
code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
|
315
|
+
code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
|
328
316
|
end
|
329
317
|
if Utils.is_response_ok?(code)
|
330
318
|
Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
|
331
|
-
|
319
|
+
chunk_notifier.tmpdata.sweep!
|
320
|
+
block_notifier.tmpdata.sweep!
|
332
321
|
end
|
333
322
|
[code, data]
|
334
323
|
end
|
data/lib/qiniu/rs/version.rb
CHANGED
data/qiniu-rs.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.version = Qiniu::RS::Version.to_s
|
18
18
|
|
19
19
|
# specify any dependencies here; for example:
|
20
|
-
gem.add_development_dependency "rake", "
|
20
|
+
gem.add_development_dependency "rake", ">= 0.9"
|
21
21
|
gem.add_development_dependency "rspec", "~> 2.11"
|
22
22
|
gem.add_development_dependency "fakeweb", "~> 1.3"
|
23
23
|
gem.add_runtime_dependency "json", "~> 1.7"
|
data/spec/qiniu/rs/up_spec.rb
CHANGED
@@ -11,7 +11,7 @@ module Qiniu
|
|
11
11
|
|
12
12
|
before :all do
|
13
13
|
@localfile = "bigfile.txt"
|
14
|
-
File.open(@localfile, "w"){|f|
|
14
|
+
File.open(@localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}}
|
15
15
|
@bucket = "up_test_bucket"
|
16
16
|
@key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s)
|
17
17
|
|
@@ -21,10 +21,8 @@ module Qiniu
|
|
21
21
|
end
|
22
22
|
|
23
23
|
after :all do
|
24
|
-
@localfile = "bigfile.txt"
|
25
24
|
File.unlink(@localfile) if File.exists?(@localfile)
|
26
25
|
|
27
|
-
@bucket = "up_test_bucket"
|
28
26
|
code, data = Qiniu::RS::RS.drop(@bucket)
|
29
27
|
puts [code, data].inspect
|
30
28
|
code.should == 200
|
data/spec/spec_helper.rb
CHANGED
@@ -6,13 +6,7 @@ require 'rspec'
|
|
6
6
|
|
7
7
|
RSpec.configure do |config|
|
8
8
|
config.before :all do
|
9
|
-
Qiniu::RS.establish_connection! :access_key => "
|
10
|
-
:secret_key => "
|
11
|
-
:auth_url => "http://m1.qbox.me:13001/oauth2/token",
|
12
|
-
:rs_host => "http://m1.qbox.me:13003",
|
13
|
-
:io_host => "http://m1.qbox.me:13004",
|
14
|
-
:up_host => "http://m1.qbox.me:13019",
|
15
|
-
:pub_host => "http://m1.qbox.me:13012",
|
16
|
-
:eu_host => "http://m1.qbox.me:13050"
|
9
|
+
Qiniu::RS.establish_connection! :access_key => "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
|
10
|
+
:secret_key => "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j"
|
17
11
|
end
|
18
12
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qiniu-rs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.2.
|
4
|
+
version: 3.2.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,14 +9,14 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-12-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
|
-
- -
|
19
|
+
- - ! '>='
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '0.9'
|
22
22
|
type: :development
|
@@ -24,7 +24,7 @@ dependencies:
|
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
none: false
|
26
26
|
requirements:
|
27
|
-
- -
|
27
|
+
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0.9'
|
30
30
|
- !ruby/object:Gem::Dependency
|
@@ -133,12 +133,12 @@ files:
|
|
133
133
|
- .gitignore
|
134
134
|
- .rspec
|
135
135
|
- .travis.yml
|
136
|
+
- CHANGELOG.md
|
136
137
|
- Gemfile
|
137
138
|
- Gemfile.lock
|
138
139
|
- LICENSE
|
139
140
|
- README.md
|
140
141
|
- Rakefile
|
141
|
-
- docs/NEWS.md
|
142
142
|
- docs/README.md
|
143
143
|
- lib/qiniu-rs.rb
|
144
144
|
- lib/qiniu/rs.rb
|
@@ -186,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
186
186
|
version: '0'
|
187
187
|
segments:
|
188
188
|
- 0
|
189
|
-
hash: -
|
189
|
+
hash: -4300609686795785448
|
190
190
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
191
191
|
none: false
|
192
192
|
requirements:
|
@@ -195,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
195
195
|
version: '0'
|
196
196
|
segments:
|
197
197
|
- 0
|
198
|
-
hash: -
|
198
|
+
hash: -4300609686795785448
|
199
199
|
requirements: []
|
200
200
|
rubyforge_project:
|
201
201
|
rubygems_version: 1.8.24
|