qiniu-rs 3.2.1 → 3.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,11 @@
1
1
  ## CHANGE LOG
2
2
 
3
+ ### v3.2.2
4
+
5
+ fixed E701 error
6
+
7
+ 断点续上传根据 mkblk 返回的 host 字段进行 bput 和 mkfile ,规避由于DNS智能解析造成的分布式并行块上传会出现上下文不连贯导致的 E701 问题。
8
+
3
9
  ### v3.2.1
4
10
 
5
11
  allow images uploaded auto-orient.
@@ -22,4 +28,4 @@ allow files uploaded auto callback some APIs (like imageInfo, exif, etc…), and
22
28
  参考:
23
29
 
24
30
  1. [[API] 生成上传授权凭证 uploadToken 之 escape 参数详解](http://docs.qiniutek.com/v3/api/io/#escape-expression)
25
- 2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
31
+ 2. [[SDK] Qiniu::RS.generate_upload_token() 方法中的参数增加了 :escape 选项](http://docs.qiniutek.com/v3/sdk/ruby/#generate-upload-token)
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- qiniu-rs (3.2.1)
4
+ qiniu-rs (3.2.2)
5
5
  json (~> 1.7)
6
6
  mime-types (~> 1.19)
7
7
  rest-client (~> 1.6)
@@ -14,17 +14,17 @@ GEM
14
14
  fakeweb (1.3.0)
15
15
  json (1.7.5)
16
16
  mime-types (1.19)
17
- rake (0.9.2.2)
17
+ rake (10.0.2)
18
18
  rest-client (1.6.7)
19
19
  mime-types (>= 1.16)
20
- rspec (2.11.0)
21
- rspec-core (~> 2.11.0)
22
- rspec-expectations (~> 2.11.0)
23
- rspec-mocks (~> 2.11.0)
24
- rspec-core (2.11.1)
25
- rspec-expectations (2.11.3)
20
+ rspec (2.12.0)
21
+ rspec-core (~> 2.12.0)
22
+ rspec-expectations (~> 2.12.0)
23
+ rspec-mocks (~> 2.12.0)
24
+ rspec-core (2.12.1)
25
+ rspec-expectations (2.12.0)
26
26
  diff-lcs (~> 1.1.3)
27
- rspec-mocks (2.11.3)
27
+ rspec-mocks (2.12.0)
28
28
  ruby-hmac (0.4.0)
29
29
 
30
30
  PLATFORMS
@@ -33,5 +33,5 @@ PLATFORMS
33
33
  DEPENDENCIES
34
34
  fakeweb (~> 1.3)
35
35
  qiniu-rs!
36
- rake (~> 0.9)
36
+ rake (>= 0.9)
37
37
  rspec (~> 2.11)
@@ -4,9 +4,6 @@ module Qiniu
4
4
  module RS
5
5
 
6
6
  class Exception < RuntimeError
7
- def to_s
8
- inspect
9
- end
10
7
  end
11
8
 
12
9
  class ResponseError < Exception
data/lib/qiniu/rs/up.rb CHANGED
@@ -14,6 +14,9 @@ module Qiniu
14
14
  module RS
15
15
  module UP
16
16
 
17
+ PROGRESS_TMP_FILE = 'progresses'
18
+ CHECKSUM_TMP_FILE = 'ctxes'
19
+
17
20
  module AbstractClass
18
21
  class ChunkProgressNotifier
19
22
  include Qiniu::RS::Abstract
@@ -29,78 +32,58 @@ module Qiniu
29
32
  end
30
33
 
31
34
  class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier
35
+ attr_reader :tmpdata
32
36
  def initialize(id)
33
- @data = UP::ProgressData.new(id)
37
+ @tmpdata = UP::TmpData.new(id, PROGRESS_TMP_FILE)
34
38
  end
35
39
  def notify(index, progress)
36
- @data.set_progresses(index, progress)
37
- logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" +
38
- "{ctx:#{progress[:ctx]}, offset:#{progress[:offset]}, restsize:#{progress[:restsize]}, status_code:#{progress[:status_code]}}"
40
+ @tmpdata.set(index, progress)
41
+ logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s
39
42
  Utils.debug(logmsg)
40
43
  end
41
44
  end
42
45
 
43
46
  class BlockProgressNotifier < AbstractClass::BlockProgressNotifier
47
+ attr_reader :tmpdata
44
48
  def initialize(id)
45
- @data = UP::ProgressData.new(id)
49
+ @tmpdata = UP::TmpData.new(id, CHECKSUM_TMP_FILE)
46
50
  end
47
51
  def notify(index, checksum)
48
- @data.set_checksums(index, checksum)
52
+ @tmpdata.set(index, checksum)
53
+ Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded."
49
54
  Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded."
50
55
  end
51
56
  end
52
57
 
53
- class ProgressData
54
- def initialize(id)
55
- @id = id
56
- @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + @id
58
+ class TmpData
59
+ def initialize(dir, filename)
60
+ @tmpdir = Config.settings[:tmpdir] + File::SEPARATOR + dir
57
61
  FileUtils.mkdir_p(@tmpdir) unless Dir.exists?(@tmpdir)
58
- @checksum_file = @tmpdir + File::SEPARATOR + 'checksums'
59
- @progress_file = @tmpdir + File::SEPARATOR + 'progresses'
60
- end
61
-
62
- def get_checksums
63
- File.exist?(@checksum_file) ? YAML.load_file(@checksum_file) : []
64
- end
65
-
66
- def get_progresses
67
- File.exist?(@progress_file) ? YAML.load_file(@progress_file) : []
62
+ @tmpfile = @tmpdir + File::SEPARATOR + filename
68
63
  end
69
64
 
70
- def set_checksums(index, checksum)
71
- checksums = get_checksums
72
- checksums[index] = checksum
73
- File.open(@checksum_file, "w") do |f|
74
- YAML::dump(checksums, f)
75
- Utils.debug %Q(Updating tmpfile: #{@checksum_file})
65
+ def init(values)
66
+ File.open(@tmpfile, "w") do |f|
67
+ YAML::dump(values, f)
68
+ Utils.debug %Q(Initializing tmpfile: #{@tmpfile})
76
69
  end
77
70
  end
78
71
 
79
- def set_progresses(index, progress)
80
- progresses = get_progresses
81
- progresses[index] = progress
82
- File.open(@progress_file, "w") do |f|
83
- YAML::dump(progresses, f)
84
- Utils.debug %Q(Updating tmpfile: #{@progress_file})
85
- end
72
+ def all
73
+ File.exist?(@tmpfile) ? YAML.load_file(@tmpfile) : []
86
74
  end
87
75
 
88
- def init_checksums(checksums)
89
- File.open(@checksum_file, "w") do |f|
90
- YAML::dump(checksums, f)
91
- Utils.debug %Q(Initializing tmpfile: #{@checksum_file})
92
- end
93
- end
94
-
95
- def init_progresses(progresses)
96
- File.open(@progress_file, "w") do |f|
97
- YAML::dump(progresses, f)
98
- Utils.debug %Q(Initializing tmpfile: #{@progress_file})
76
+ def set(index, value)
77
+ values = all
78
+ values[index] = value
79
+ File.open(@tmpfile, "w") do |f|
80
+ YAML::dump(values, f)
81
+ Utils.debug %Q(Updating tmpfile: #{@tmpfile})
99
82
  end
100
83
  end
101
84
 
102
85
  def sweep!
103
- FileUtils.rm_r(@tmpdir)
86
+ FileUtils.rm_r(@tmpdir) if Dir.exists?(@tmpdir)
104
87
  end
105
88
  end
106
89
 
@@ -162,20 +145,21 @@ module Qiniu
162
145
  end
163
146
 
164
147
  def _new_block_put_progress_data
165
- {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil}
148
+ {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
166
149
  end
167
150
 
168
- def _call_binary_with_token(uptoken, url, data, retry_times = 0)
151
+ def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
169
152
  options = {
170
153
  :method => :post,
171
154
  :content_type => 'application/octet-stream',
172
155
  :upload_signature_token => uptoken
173
156
  }
157
+ options[:content_type] = content_type if !content_type.nil? && !content_type.empty?
174
158
  code, data = http_request url, data, options
175
159
  unless Utils.is_response_ok?(code)
176
160
  retry_times += 1
177
161
  if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
178
- return _call_binary_with_token(uptoken, url, data, retry_times)
162
+ return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
179
163
  end
180
164
  end
181
165
  [code, data]
@@ -186,8 +170,8 @@ module Qiniu
186
170
  _call_binary_with_token(uptoken, url, body)
187
171
  end
188
172
 
189
- def _putblock(uptoken, ctx, offset, body)
190
- url = Config.settings[:up_host] + "/bput/#{ctx}/#{offset}"
173
+ def _putblock(uphost, uptoken, ctx, offset, body)
174
+ url = uphost + "/bput/#{ctx}/#{offset}"
191
175
  _call_binary_with_token(uptoken, url, body)
192
176
  end
193
177
 
@@ -214,6 +198,7 @@ module Qiniu
214
198
  progress[:offset] = body_length
215
199
  progress[:restsize] = block_size - body_length
216
200
  progress[:status_code] = code
201
+ progress[:host] = data["host"]
217
202
  if !notifier.nil? && notifier.respond_to?("notify")
218
203
  notifier.notify(block_index, progress)
219
204
  end
@@ -236,13 +221,14 @@ module Qiniu
236
221
  if result_length != body_length
237
222
  raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
238
223
  end
239
- code, data = _putblock(uptoken, progress[:ctx], progress[:offset], body)
224
+ code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
240
225
  body_crc32 = Zlib.crc32(body)
241
226
  if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
242
227
  progress[:ctx] = data["ctx"]
243
228
  progress[:offset] += body_length
244
229
  progress[:restsize] -= body_length
245
230
  progress[:status_code] = code
231
+ progress[:host] = data["host"]
246
232
  if !notifier.nil? && notifier.respond_to?("notify")
247
233
  notifier.notify(block_index, progress)
248
234
  end
@@ -280,7 +266,8 @@ module Qiniu
280
266
  end
281
267
  code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
282
268
  if Utils.is_response_ok?(code)
283
- checksums[block_index] = data["checksum"]
269
+ #checksums[block_index] = data["checksum"]
270
+ checksums[block_index] = data["ctx"]
284
271
  if !block_notifier.nil? && block_notifier.respond_to?("notify")
285
272
  block_notifier.notify(block_index, checksums[block_index])
286
273
  end
@@ -290,7 +277,7 @@ module Qiniu
290
277
  return [code, data]
291
278
  end
292
279
 
293
- def _mkfile(uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
280
+ def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
294
281
  path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
295
282
  path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
296
283
  path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
@@ -298,37 +285,39 @@ module Qiniu
298
285
  callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
299
286
  path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
300
287
  path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
301
- url = Config.settings[:up_host] + path
302
- body = ''
303
- checksums.each do |checksum|
304
- body += Utils.urlsafe_base64_decode(checksum)
305
- end
306
- _call_binary_with_token(uptoken, url, body)
288
+ url = uphost + path
289
+ #body = ''
290
+ #checksums.each do |checksum|
291
+ # body += Utils.urlsafe_base64_decode(checksum)
292
+ #end
293
+ body = checksums.join(',')
294
+ _call_binary_with_token(uptoken, url, body, 'text/plain')
307
295
  end
308
296
 
309
297
  def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
310
298
  block_count = _block_count(fsize)
311
- progress_data = ProgressData.new(key)
312
- checksums = progress_data.get_checksums
313
- progresses = progress_data.get_progresses
314
- if checksums.empty?
315
- block_count.times{checksums << ''}
316
- progress_data.init_checksums(checksums)
317
- end
299
+ chunk_notifier = ChunkProgressNotifier.new(key)
300
+ block_notifier = BlockProgressNotifier.new(key)
301
+ progresses = chunk_notifier.tmpdata.all
318
302
  if progresses.empty?
319
303
  block_count.times{progresses << _new_block_put_progress_data}
320
- progress_data.init_progresses(progresses)
304
+ chunk_notifier.tmpdata.init(progresses)
305
+ end
306
+ checksums = block_notifier.tmpdata.all
307
+ if checksums.empty?
308
+ block_count.times{checksums << ''}
309
+ block_notifier.tmpdata.init(checksums)
321
310
  end
322
- chunk_notifier = ChunkProgressNotifier.new(key)
323
- block_notifier = BlockProgressNotifier.new(key)
324
311
  code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
325
312
  if Utils.is_response_ok?(code)
313
+ uphost = data["host"]
326
314
  entry_uri = bucket + ':' + key
327
- code, data = _mkfile(uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
315
+ code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
328
316
  end
329
317
  if Utils.is_response_ok?(code)
330
318
  Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
331
- progress_data.sweep!
319
+ chunk_notifier.tmpdata.sweep!
320
+ block_notifier.tmpdata.sweep!
332
321
  end
333
322
  [code, data]
334
323
  end
@@ -5,7 +5,7 @@ module Qiniu
5
5
  module Version
6
6
  MAJOR = 3
7
7
  MINOR = 2
8
- PATCH = 1
8
+ PATCH = 2
9
9
  # Returns a version string by joining <tt>MAJOR</tt>, <tt>MINOR</tt>, and <tt>PATCH</tt> with <tt>'.'</tt>
10
10
  #
11
11
  # Example
data/qiniu-rs.gemspec CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
17
17
  gem.version = Qiniu::RS::Version.to_s
18
18
 
19
19
  # specify any dependencies here; for example:
20
- gem.add_development_dependency "rake", "~> 0.9"
20
+ gem.add_development_dependency "rake", ">= 0.9"
21
21
  gem.add_development_dependency "rspec", "~> 2.11"
22
22
  gem.add_development_dependency "fakeweb", "~> 1.3"
23
23
  gem.add_runtime_dependency "json", "~> 1.7"
@@ -11,7 +11,7 @@ module Qiniu
11
11
 
12
12
  before :all do
13
13
  @localfile = "bigfile.txt"
14
- File.open(@localfile, "w"){|f| 9437184.times{f.write(Random.rand(9).to_s)}}
14
+ File.open(@localfile, "w"){|f| 5242888.times{f.write(Random.rand(9).to_s)}}
15
15
  @bucket = "up_test_bucket"
16
16
  @key = Digest::SHA1.hexdigest(@localfile+Time.now.to_s)
17
17
 
@@ -21,10 +21,8 @@ module Qiniu
21
21
  end
22
22
 
23
23
  after :all do
24
- @localfile = "bigfile.txt"
25
24
  File.unlink(@localfile) if File.exists?(@localfile)
26
25
 
27
- @bucket = "up_test_bucket"
28
26
  code, data = Qiniu::RS::RS.drop(@bucket)
29
27
  puts [code, data].inspect
30
28
  code.should == 200
data/spec/spec_helper.rb CHANGED
@@ -6,13 +6,7 @@ require 'rspec'
6
6
 
7
7
  RSpec.configure do |config|
8
8
  config.before :all do
9
- Qiniu::RS.establish_connection! :access_key => "dFX_wMGVrRzwdWaraW-Qe5ZCDT-kcSmIAGKQOkXh",
10
- :secret_key => "VllxxDfkn_h2ZIqeKYTnHJiN4LVODfDBlJHy_KsW",
11
- :auth_url => "http://m1.qbox.me:13001/oauth2/token",
12
- :rs_host => "http://m1.qbox.me:13003",
13
- :io_host => "http://m1.qbox.me:13004",
14
- :up_host => "http://m1.qbox.me:13019",
15
- :pub_host => "http://m1.qbox.me:13012",
16
- :eu_host => "http://m1.qbox.me:13050"
9
+ Qiniu::RS.establish_connection! :access_key => "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
10
+ :secret_key => "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j"
17
11
  end
18
12
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qiniu-rs
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.1
4
+ version: 3.2.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,14 +9,14 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-11-07 00:00:00.000000000 Z
12
+ date: 2012-12-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
- - - ~>
19
+ - - ! '>='
20
20
  - !ruby/object:Gem::Version
21
21
  version: '0.9'
22
22
  type: :development
@@ -24,7 +24,7 @@ dependencies:
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  none: false
26
26
  requirements:
27
- - - ~>
27
+ - - ! '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: '0.9'
30
30
  - !ruby/object:Gem::Dependency
@@ -133,12 +133,12 @@ files:
133
133
  - .gitignore
134
134
  - .rspec
135
135
  - .travis.yml
136
+ - CHANGELOG.md
136
137
  - Gemfile
137
138
  - Gemfile.lock
138
139
  - LICENSE
139
140
  - README.md
140
141
  - Rakefile
141
- - docs/NEWS.md
142
142
  - docs/README.md
143
143
  - lib/qiniu-rs.rb
144
144
  - lib/qiniu/rs.rb
@@ -186,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
186
186
  version: '0'
187
187
  segments:
188
188
  - 0
189
- hash: -726479024348899519
189
+ hash: -4300609686795785448
190
190
  required_rubygems_version: !ruby/object:Gem::Requirement
191
191
  none: false
192
192
  requirements:
@@ -195,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
195
195
  version: '0'
196
196
  segments:
197
197
  - 0
198
- hash: -726479024348899519
198
+ hash: -4300609686795785448
199
199
  requirements: []
200
200
  rubyforge_project:
201
201
  rubygems_version: 1.8.24