pindo 4.7.6 → 4.7.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d67d718e2507fa330a19b44df9b7abefb0ae417dac58a4c5d431cf3c9f63dc13
4
- data.tar.gz: 3377d33ac73c5c5024beb48de2fa30d0b1813f0637eae2d990a30eb0ae37f9f7
3
+ metadata.gz: 9a9d88e4cefb78e407a08819025c3e4e04ddca1b33dfdbaa2910643ca0314f9a
4
+ data.tar.gz: c4870fd7e8b90626cf398b48bfb55b432ce4e867bd2737ae1b8793056353bb0e
5
5
  SHA512:
6
- metadata.gz: '01038d7dfe0a9b0705056798db4cdbd6897baadb944c9d9808910526a7dead73d5f5573efa73bd0ef9d7dad5dd0e14c28628046b5dbb658571f5865afff2cf86'
7
- data.tar.gz: c23e4788703dc11aa106d94c12c8c28ade2d5e0c72241b77bd75263b1e36115abb0a4ff1b633d37ccd9a784f51252f45e1ea8899ce676a91e832b775d386b760
6
+ metadata.gz: 9f30678026ced27fe6c692449d26cd2518d5ac8f83e410ef5c7382c475b62140e2c6e9939e24388d32235d10554aca61bf48a113f7df82b94193d2bff8f72a29
7
+ data.tar.gz: 314c346c4393769e9b2e98b65fbee1d28f8ff373215422adb5c27f327dbd610faa9b917e8a684dcc0bb37c72fcd45161cab32e9a6f9e7f1e08a2288c49ecb01c
@@ -20,6 +20,10 @@ module Pindo
20
20
  attr_accessor :default_url
21
21
  attr_accessor :attach_url
22
22
 
23
+ attr_accessor :upload_binary_file
24
+ attr_accessor :file_size
25
+ attr_accessor :progress_bar
26
+
23
27
 
24
28
  def initialize()
25
29
 
@@ -68,15 +72,17 @@ module Pindo
68
72
 
69
73
  def upload_file(binary_file:nil, isAttach:false)
70
74
 
71
- extension = File.extname(binary_file)
72
- filename = File.basename(binary_file)
73
- filesize = File.size(binary_file)
75
+ @upload_binary_file = binary_file
76
+ @file_size = File.size(@upload_binary_file)
77
+ @progress_bar = PgyerUploadProgressBar.new(upload_total_size:@file_size)
78
+
79
+ extension = File.extname(@upload_binary_file)
80
+ filename = File.basename(@upload_binary_file)
74
81
  size_level = 1024 * 1024
75
- file_bytes = File.binread(binary_file)
82
+ file_bytes = File.binread(@upload_binary_file)
76
83
  checksum = Digest::MD5.hexdigest(file_bytes)
77
84
  file_uuid = SecureRandom.uuid
78
- total_m = sprintf("%.2f", 1.00 * filesize / 1024 /1024 )
79
-
85
+ total_m = sprintf("%.2f", 1.00 * @file_size / 1024 /1024 )
80
86
 
81
87
 
82
88
  upload_path = @default_url
@@ -86,92 +92,183 @@ module Pindo
86
92
  content_disposition = "attachment; filename=#{filename}"
87
93
  end
88
94
 
89
- object_key = upload_path + file_uuid + extension
95
+ upload_path_key = upload_path + file_uuid + extension
90
96
 
91
- puts "文件路径: #{binary_file}"
97
+ task_num = 5
98
+
99
+ puts "文件路径: #{@upload_binary_file}"
92
100
  puts "文件大小: #{total_m}M"
93
- puts "上传路径: #{object_key}"
101
+ puts "上传路径: #{upload_path_key}"
102
+ puts "切片大小: 5 MB"
103
+ puts "最大线程数: #{task_num}"
94
104
  puts
95
105
 
96
-
97
- @upload_total_size = filesize
98
-
99
-
100
106
  upload_result = nil
101
107
 
102
- file_size_param = 1.00 * filesize / 1024 /1024
103
- result_data = post_upload_url_req(upload_key:object_key, file_size:file_size_param.ceil)
108
+ file_size_param = 1.00 * @file_size / 1024 /1024
109
+ result_data = post_upload_url_req(upload_path_key:upload_path_key, file_ceil_size:file_size_param.ceil)
104
110
 
105
111
  upload_id= result_data["data"]["uploadId"]
106
- upload_params_list = result_data["data"]["uploadParamsList"]
112
+ @upload_params_list = result_data["data"]["uploadParamsList"]
107
113
 
108
-
109
114
 
110
115
  Funlog.instance.fancyinfo_start("开始上传...")
116
+ @upload_eTags = []
117
+
118
+ # single_task_upload_data_req( )
111
119
 
112
- eTags = []
113
- eTags = single_task_upload_data_req(binary_file:binary_file, upload_params_list:upload_params_list)
114
120
 
115
- result_data = post_upload_finish_req(upload_key:object_key, upload_id:upload_id, eTags:eTags)
121
+ multi_task_upload_data_req(task_num:task_num)
116
122
 
123
+ result_data = post_upload_finish_req(upload_path_key:upload_path_key, upload_id:upload_id, eTags:@upload_eTags)
117
124
  if result_data["code"] == 200
118
- upload_result = object_key
119
- # set_progress(filesize, filesize)
120
- Funlog.instance.fancyinfo_success("文件#{binary_file} 上传成功! 😎😎😎")
125
+ upload_result = upload_path_key
126
+ Funlog.instance.fancyinfo_success("文件#{@upload_binary_file} 上传成功! 😎😎😎")
121
127
  else
122
128
  upload_result = nil
123
- Funlog.instance.fancyinfo_error("文件#{binary_file} 上传失败! 😭😭😭")
129
+ Funlog.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
124
130
  end
125
131
 
126
132
  return upload_result
127
133
 
128
134
  end
129
135
 
130
- def single_task_upload_data_req(binary_file:nil, upload_params_list:nil)
131
- filesize = File.size(binary_file)
132
- eTags = []
133
- if !upload_params_list.nil? && upload_params_list.length > 0
134
- for i in 0..upload_params_list.length-1 do
135
- part_result = single_task_upload_part_data_req(filesize:filesize, binary_file:binary_file, upload_url:upload_params_list[i]["signedUrl"], part_no:upload_params_list[i]["partNo"])
136
- eTag_item = { partNumber: upload_params_list[i]["partNo"], tag: part_result}
137
- eTags << eTag_item
136
+
137
+ def multi_task_upload_data_req(task_num:1)
138
+
139
+ # puts "add task_num :#{task_num}"
140
+
141
+ for i in 0..@upload_params_list.length-1 do
142
+ @upload_params_list[i]["retryCount"] = 3
143
+ end
144
+
145
+ while @upload_params_list.size > 0
146
+ upload_params_list_temp = []
147
+ #每次最大5个线程上传
148
+ for i in 0..task_num do
149
+ upload_params_list_temp << @upload_params_list.shift
150
+ end
151
+
152
+ hydra = Typhoeus::Hydra.new
153
+ while upload_params_list_temp.size > 0
154
+ upload_params_item = upload_params_list_temp.shift
155
+ unless upload_params_item.nil?
156
+ single_task_upload_part_data_req(upload_params_item:upload_params_item, hydra_handle:hydra)
157
+ end
138
158
  end
159
+ hydra.run
160
+ end
161
+
162
+ end
163
+
164
+ def single_task_upload_data_req()
165
+
166
+ for i in 0..@upload_params_list.length-1 do
167
+ @upload_params_list[i]["retryCount"] = 3
168
+ end
169
+
170
+ while @upload_params_list.size > 0
171
+ upload_params_item = @upload_params_list.shift
172
+ single_task_upload_part_data_req(upload_params_item:upload_params_item)
139
173
  end
140
174
 
141
- return eTags
175
+ # if !@upload_params_list.nil? && @upload_params_list.length > 0
176
+ # for i in 0..@upload_params_list.length-1 do
177
+ # eTag_item = single_task_upload_part_data_req(upload_params_item:@upload_params_list[i])
178
+ # end
179
+ # end
180
+
181
+ return @eTags
142
182
  end
143
183
 
144
- def single_task_upload_part_data_req(filesize:nil, binary_file:nil, upload_url:nil, part_no:nil)
184
+ def single_task_upload_part_data_req(upload_params_item:nil, hydra_handle:nil)
145
185
 
146
- # puts "start upload part #{part_no} -----"
186
+ upload_url = upload_params_item["signedUrl"]
187
+ part_no = upload_params_item["partNo"]
147
188
 
148
189
  file_size_ele = 1024 * 1024 * 5 #5M
149
190
  start_position = file_size_ele * (part_no -1)
150
- if part_no * file_size_ele > filesize
151
- read_length = filesize - start_position
191
+ if part_no * file_size_ele > @file_size
192
+ read_length = @file_size - start_position
152
193
  else
153
194
  read_length = file_size_ele
154
195
  end
155
196
 
156
- file = File.open(binary_file, "rb")
197
+ file = File.open(@upload_binary_file, "rb")
157
198
  file.seek(start_position)
158
199
  put_data = file.read(read_length)
159
200
 
160
-
201
+ request = create_req(upload_url:upload_url, body_data:put_data, read_length:read_length)
202
+
203
+ # @upload_num = start_position
204
+
205
+ request.on_progress do |dltotal, dlnow, ultotal, ulnow|
206
+ # puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})"
207
+ if ulnow
208
+ # @upload_num = start_position + ulnow
209
+ # set_progress(@upload_num, @file_size)
210
+ @progress_bar.update_upload_index(upload_part:part_no, upload_size:ulnow)
211
+ @progress_bar.update_upload_progress()
212
+ end
213
+ end
214
+
215
+ eTag_item = nil
216
+
217
+ request.on_failure do |response|
218
+ @progress_bar.delete_upload_index(upload_part:part_no)
219
+ upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
220
+ if upload_params_item["retryCount"] > 0
221
+ @upload_params_list.push(upload_params_item)
222
+ else
223
+ Funlog.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
224
+ raise Informative "上传文件失败"
225
+ end
226
+ end
227
+
228
+ request.on_complete do |response|
229
+ if response.success?
230
+ @progress_bar.complete_upload_index(upload_part:part_no, complete_size:read_length)
231
+ etag = response.headers["ETag"]
232
+ eTag_item = { partNumber: part_no, tag: etag}
233
+ @upload_eTags << eTag_item
234
+ else
235
+ @progress_bar.delete_upload_index(upload_part:part_no)
236
+ upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
237
+ if upload_params_item["retryCount"] > 0
238
+ @upload_params_list.push(upload_params_item)
239
+ else
240
+ Funlog.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
241
+ raise Informative "上传文件失败"
242
+ end
243
+ end
244
+ end
245
+
246
+ if hydra_handle.nil?
247
+ request.run
248
+ else
249
+ hydra_handle.queue(request)
250
+ end
251
+
252
+ return eTag_item
253
+
254
+ end
255
+
256
+ def create_req(upload_url:nil, body_data:nil, read_length:nil)
257
+
161
258
  request = nil
162
259
 
163
260
  proxy_options = {
164
- proxy: ENV['http_proxy'] || ENV['https_proxy'],
261
+ proxy: ENV['http_proxy'] || ENV['https_proxy'],
165
262
  proxyuserpwd: "#{ENV['HTTP_PROXY_USER']}:#{ENV['HTTP_PROXY_PASSWORD']}"
166
263
  }
167
-
264
+
168
265
  if proxy_options[:proxy]
169
266
  request = Typhoeus::Request.new(
170
267
  upload_url,
171
268
  method: :put,
172
269
  proxy: proxy_options[:proxy],
173
270
  proxyuserpwd: proxy_options[:proxyuserpwd],
174
- :body => put_data,
271
+ :body => body_data,
175
272
  :headers => {
176
273
  'Content-Type' => 'application/octet-stream',
177
274
  'token' => @token['token'],
@@ -182,48 +279,25 @@ module Pindo
182
279
  request = Typhoeus::Request.new(
183
280
  upload_url,
184
281
  method: :put,
185
- :body => put_data,
282
+ :body => body_data,
186
283
  :headers => {
187
284
  'Content-Type' => 'application/octet-stream',
188
285
  'token' => @token['token'],
189
286
  'Content-Length' => read_length.to_s
190
287
  }
191
288
  )
192
- end
193
-
194
-
195
- @upload_num = start_position
196
- request.on_progress do |dltotal, dlnow, ultotal, ulnow|
197
- # puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})"
198
- if ulnow
199
- @upload_num = start_position + ulnow
200
- set_progress(@upload_num, filesize)
201
- end
202
- end
203
289
 
204
- return_data = nil
205
- request.on_complete do |response|
206
- if response.success?
207
- # puts "on_complete success"
208
- return_data = response.headers["ETag"]
209
- else
210
- # puts "on_complete fail"
211
- end
212
290
  end
213
-
214
- request.run
215
-
216
- return return_data
217
-
291
+ return request
218
292
  end
219
293
 
220
- def post_upload_finish_req(upload_key:nil, upload_id:nil, eTags:nil)
294
+ def post_upload_finish_req(upload_path_key:nil, upload_id:nil, eTags:nil)
221
295
 
222
296
  boss_url = @baseurl + @request_config["multi_signed_url_upload"]
223
297
 
224
298
  body_params = {
225
299
  functionName:"finish",
226
- fileKey:upload_key,
300
+ fileKey:upload_path_key,
227
301
  uploadId:upload_id,
228
302
  tags:eTags
229
303
  }
@@ -247,16 +321,14 @@ module Pindo
247
321
  end
248
322
 
249
323
 
250
- def post_upload_url_req(upload_key:nil, file_size:nil)
324
+ def post_upload_url_req(upload_path_key:nil, file_ceil_size:nil)
251
325
 
252
- # puts "post_upload_url_req start ..."
253
- # puts "file_size: #{file_size} MB"
254
326
  boss_url = @baseurl + @request_config["multi_signed_url_upload"]
255
327
 
256
328
  body_params = {
257
329
  functionName:"start",
258
- fileKey:upload_key,
259
- fileSize:file_size
330
+ fileKey:upload_path_key,
331
+ fileSize:file_ceil_size
260
332
  }
261
333
 
262
334
  con = HttpClient.create_instance_with_proxy
@@ -273,19 +345,71 @@ module Pindo
273
345
  result_data = JSON.parse(res.body)
274
346
  end
275
347
 
276
- # puts JSON.pretty_generate(result_data)
277
-
278
348
  return result_data
279
349
 
280
350
  end
281
351
 
282
- def set_progress(index_num, total_num, char ='>' )
283
- progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
284
- total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
285
- index = 40.0 * index_num / total_num
286
- upload_message = "已上传:#{progress_str}\%【" + (char * (index/1).floor).ljust(40.0, '_') + "】Total: #{total_size} M"
287
- Funlog.instance.fancyinfo_update(upload_message)
352
+ class PgyerUploadProgressBar
353
+
354
+ attr_accessor :draw_char
355
+ attr_accessor :complete_size
356
+ attr_accessor :upload_total_size
357
+ attr_accessor :last_update_time
358
+ attr_accessor :update_ing_size
359
+
360
+ def initialize(upload_total_size:nil, draw_char:'>')
361
+
362
+
363
+ @upload_total_size = upload_total_size
364
+ @draw_char = draw_char
365
+ @last_update_time = (Time.now.to_f * 1000).to_i #毫秒
366
+
367
+ @complete_size = 0
368
+ @update_ing_size = {}
369
+ end
370
+
371
+ def update_upload_index(upload_part:nil, upload_size:nil)
372
+ @update_ing_size[upload_part] = upload_size
373
+ end
374
+
375
+ def delete_upload_index(upload_part:nil)
376
+ @update_ing_size[upload_part] = 0
377
+ end
378
+
379
+ def complete_upload_index(upload_part:nil, complete_size:nil)
380
+ @complete_size = @complete_size + complete_size
381
+ @update_ing_size[upload_part] = 0
382
+ end
383
+
384
+ def update_upload_progress()
385
+ time_now = (Time.now.to_f * 1000).to_i #毫秒
386
+ if time_now - @last_update_time > 80
387
+ @last_update_time = time_now
388
+ total_num = @upload_total_size
389
+ index_num = @complete_size
390
+ @update_ing_size.each do |key, value|
391
+ index_num = index_num + value
392
+ end
393
+
394
+ progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
395
+ total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
396
+ upload_size = sprintf("%.2f", 1.00 * index_num / 1024 /1024 )
397
+ index = 40.0 * index_num / total_num
398
+ upload_message = "已上传:#{upload_size}MB|#{progress_str}\%【" + (@draw_char * (index/1).floor).ljust(40.0, '_') + "】Total:#{total_size}MB"
399
+ Funlog.instance.fancyinfo_update(upload_message)
400
+ end
401
+ end
402
+
288
403
  end
289
404
 
405
+ # def set_progress(index_num, total_num, char ='>' )
406
+ # progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
407
+ # total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
408
+ # upload_size = sprintf("%.2f", 1.00 * index_num / 1024 /1024 )
409
+ # index = 40.0 * index_num / total_num
410
+ # upload_message = "已上传:#{upload_size}MB|#{progress_str}\%【" + (char * (index/1).floor).ljust(40.0, '_') + "】Total:#{total_size}MB"
411
+ # Funlog.instance.fancyinfo_update(upload_message)
412
+ # end
413
+
290
414
  end
291
415
  end
data/lib/pindo/version.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module Pindo
2
2
 
3
- VERSION = "4.7.6"
3
+ VERSION = "4.7.8"
4
4
 
5
5
  class VersionCheck
6
6
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pindo
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.7.6
4
+ version: 4.7.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - wade
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-05-22 00:00:00.000000000 Z
11
+ date: 2024-05-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: claide
@@ -134,22 +134,16 @@ dependencies:
134
134
  name: faraday-retry
135
135
  requirement: !ruby/object:Gem::Requirement
136
136
  requirements:
137
- - - "~>"
138
- - !ruby/object:Gem::Version
139
- version: 2.2.0
140
137
  - - ">="
141
138
  - !ruby/object:Gem::Version
142
- version: 2.2.0
139
+ version: '0'
143
140
  type: :runtime
144
141
  prerelease: false
145
142
  version_requirements: !ruby/object:Gem::Requirement
146
143
  requirements:
147
- - - "~>"
148
- - !ruby/object:Gem::Version
149
- version: 2.2.0
150
144
  - - ">="
151
145
  - !ruby/object:Gem::Version
152
- version: 2.2.0
146
+ version: '0'
153
147
  - !ruby/object:Gem::Dependency
154
148
  name: typhoeus
155
149
  requirement: !ruby/object:Gem::Requirement