pindo 4.7.7 → 4.7.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76171ef66d029d57c3591ee9a61116a4c6f3949fa6ea6e2127aba8df8518507f
-  data.tar.gz: 122afc9e684b1d41f62d52813e58df79d5db509c41b0767eb544a3782309f30b
+  metadata.gz: 9a9d88e4cefb78e407a08819025c3e4e04ddca1b33dfdbaa2910643ca0314f9a
+  data.tar.gz: c4870fd7e8b90626cf398b48bfb55b432ce4e867bd2737ae1b8793056353bb0e
 SHA512:
-  metadata.gz: ea060cef3c2070db870091c96fa34607bd68068e77e1e21ea289344354770353c229c2bce0f07cb4d6300b585228695e7db24a3fd78cc83bfd5a3f416a7b66ed
-  data.tar.gz: 6170b21c179abf3d5192d402437f4105c2466324ac1a003d58d7409cb91ed4629a9759e74c68d645c745a912136dfadd6eae48d7cd7c9247ed183670fa38648d
+  metadata.gz: 9f30678026ced27fe6c692449d26cd2518d5ac8f83e410ef5c7382c475b62140e2c6e9939e24388d32235d10554aca61bf48a113f7df82b94193d2bff8f72a29
+  data.tar.gz: 314c346c4393769e9b2e98b65fbee1d28f8ff373215422adb5c27f327dbd610faa9b917e8a684dcc0bb37c72fcd45161cab32e9a6f9e7f1e08a2288c49ecb01c

data/lib/pindo/client/pgyeruploadclient.rb

@@ -20,6 +20,10 @@ module Pindo
       attr_accessor :default_url
       attr_accessor :attach_url
 
+      attr_accessor :upload_binary_file
+      attr_accessor :file_size
+      attr_accessor :progress_bar
+
 
       def initialize()
 
@@ -68,15 +72,17 @@ module Pindo
 
       def upload_file(binary_file:nil, isAttach:false)
 
-        extension = File.extname(binary_file)
-        filename = File.basename(binary_file)
-        filesize = File.size(binary_file)
+        @upload_binary_file = binary_file
+        @file_size = File.size(@upload_binary_file)
+        @progress_bar = PgyerUploadProgressBar.new(upload_total_size:@file_size)
+
+        extension = File.extname(@upload_binary_file)
+        filename = File.basename(@upload_binary_file)
         size_level = 1024 * 1024
-        file_bytes = File.binread(binary_file)
+        file_bytes = File.binread(@upload_binary_file)
         checksum = Digest::MD5.hexdigest(file_bytes)
         file_uuid = SecureRandom.uuid
-        total_m = sprintf("%.2f", 1.00 * filesize / 1024 /1024 )
-
+        total_m = sprintf("%.2f", 1.00 * @file_size / 1024 /1024 )
 
 
         upload_path = @default_url
@@ -86,92 +92,183 @@ module Pindo
             content_disposition = "attachment; filename=#{filename}"
         end
 
-        object_key = upload_path + file_uuid + extension
+        upload_path_key = upload_path + file_uuid + extension
 
-        puts "文件路径: #{binary_file}"
+        task_num = 5
+
+        puts "文件路径: #{@upload_binary_file}"
         puts "文件大小: #{total_m}M"
-        puts "上传路径: #{object_key}"
+        puts "上传路径: #{upload_path_key}"
+        puts "切片大小: 5 MB"
+        puts "最大线程数: #{task_num}"
         puts
 
-
-        @upload_total_size = filesize
-
-
         upload_result = nil
 
-        file_size_param = 1.00 * filesize / 1024 /1024
-        result_data = post_upload_url_req(upload_key:object_key, file_size:file_size_param.ceil)
+        file_size_param = 1.00 * @file_size / 1024 /1024
+        result_data = post_upload_url_req(upload_path_key:upload_path_key, file_ceil_size:file_size_param.ceil)
 
         upload_id= result_data["data"]["uploadId"]
-        upload_params_list = result_data["data"]["uploadParamsList"]
+        @upload_params_list = result_data["data"]["uploadParamsList"]
 
-        
 
         Funlog.instance.fancyinfo_start("开始上传...")
+        @upload_eTags = []
+
+        # single_task_upload_data_req( )
 
-        eTags = []
-        eTags = single_task_upload_data_req(binary_file:binary_file, upload_params_list:upload_params_list)
 
-        result_data = post_upload_finish_req(upload_key:object_key, upload_id:upload_id, eTags:eTags)
+        multi_task_upload_data_req(task_num:task_num)
 
+        result_data = post_upload_finish_req(upload_path_key:upload_path_key, upload_id:upload_id, eTags:@upload_eTags)
         if result_data["code"] == 200
-            upload_result = object_key
-            # set_progress(filesize, filesize)
-            Funlog.instance.fancyinfo_success("文件#{binary_file} 上传成功! 😎😎😎")
+            upload_result = upload_path_key
+            Funlog.instance.fancyinfo_success("文件#{@upload_binary_file} 上传成功! 😎😎😎")
         else
             upload_result = nil
-            Funlog.instance.fancyinfo_error("文件#{binary_file} 上传失败! 😭😭😭")
+            Funlog.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
         end
 
         return upload_result
 
       end
 
-      def single_task_upload_data_req(binary_file:nil, upload_params_list:nil)
-        filesize = File.size(binary_file)
-        eTags = []
-        if !upload_params_list.nil? && upload_params_list.length > 0
-            for i in 0..upload_params_list.length-1 do
-                part_result = single_task_upload_part_data_req(filesize:filesize, binary_file:binary_file, upload_url:upload_params_list[i]["signedUrl"], part_no:upload_params_list[i]["partNo"])
-                eTag_item = { partNumber: upload_params_list[i]["partNo"], tag: part_result}
-                eTags << eTag_item
+
+      def multi_task_upload_data_req(task_num:1)
+
+          # puts "add task_num :#{task_num}"
+
+          for i in 0..@upload_params_list.length-1 do
+            @upload_params_list[i]["retryCount"] = 3
+          end
+
+          while @upload_params_list.size > 0
+            upload_params_list_temp = []
+            #每次最大5个线程上传
+            for i in 0..task_num do
+              upload_params_list_temp << @upload_params_list.shift
+            end
+
+            hydra = Typhoeus::Hydra.new
+            while upload_params_list_temp.size > 0
+                upload_params_item = upload_params_list_temp.shift
+                unless upload_params_item.nil?
+                    single_task_upload_part_data_req(upload_params_item:upload_params_item, hydra_handle:hydra)
+                end
             end
+            hydra.run
+          end
+
+      end
+
+      def single_task_upload_data_req()
+
+        for i in 0..@upload_params_list.length-1 do
+          @upload_params_list[i]["retryCount"] = 3
+        end
+
+        while @upload_params_list.size > 0
+          upload_params_item = @upload_params_list.shift
+          single_task_upload_part_data_req(upload_params_item:upload_params_item)
         end
 
-        return eTags
+        # if !@upload_params_list.nil? && @upload_params_list.length > 0
+        #     for i in 0..@upload_params_list.length-1 do
+        #         eTag_item = single_task_upload_part_data_req(upload_params_item:@upload_params_list[i])
+        #     end
+        # end
+
+        return @eTags
       end
 
-      def single_task_upload_part_data_req(filesize:nil, binary_file:nil, upload_url:nil, part_no:nil)
+      def single_task_upload_part_data_req(upload_params_item:nil, hydra_handle:nil)
 
-          # puts "start upload part #{part_no} -----"
+          upload_url = upload_params_item["signedUrl"]
+          part_no = upload_params_item["partNo"]
 
           file_size_ele = 1024 * 1024 * 5  #5M
           start_position = file_size_ele * (part_no -1)
-          if part_no * file_size_ele > filesize
-            read_length = filesize - start_position
+          if part_no * file_size_ele > @file_size
+            read_length = @file_size - start_position
           else
             read_length = file_size_ele
           end
 
-          file = File.open(binary_file, "rb")
+          file = File.open(@upload_binary_file, "rb")
           file.seek(start_position)
           put_data = file.read(read_length)
 
-        
+          request = create_req(upload_url:upload_url, body_data:put_data, read_length:read_length)
+
+          # @upload_num = start_position
+
+          request.on_progress do |dltotal, dlnow, ultotal, ulnow|
+              # puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})"
+              if ulnow
+                # @upload_num = start_position + ulnow
+                # set_progress(@upload_num, @file_size)
+                @progress_bar.update_upload_index(upload_part:part_no, upload_size:ulnow)
+                @progress_bar.update_upload_progress()
+              end
+          end
+
+          eTag_item = nil
+
+          request.on_failure do |response|
+              @progress_bar.delete_upload_index(upload_part:part_no)
+              upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
+              if upload_params_item["retryCount"] > 0
+                @upload_params_list.push(upload_params_item)
+              else
+                Funlog.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
+                raise Informative "上传文件失败"
+              end
+          end
+
+          request.on_complete do |response|
+              if response.success?
+                @progress_bar.complete_upload_index(upload_part:part_no, complete_size:read_length)
+                etag = response.headers["ETag"]
+                eTag_item = { partNumber: part_no, tag: etag}
+                @upload_eTags << eTag_item
+              else
+                @progress_bar.delete_upload_index(upload_part:part_no)
+                upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
+                if upload_params_item["retryCount"] > 0
+                  @upload_params_list.push(upload_params_item)
+                else
+                  Funlog.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
+                  raise Informative "上传文件失败"
+                end
+              end
+          end
+
+          if hydra_handle.nil?
+            request.run
+          else
+            hydra_handle.queue(request)
+          end
+
+          return  eTag_item
+
+      end
+
+      def create_req(upload_url:nil, body_data:nil, read_length:nil)
+
           request = nil
 
           proxy_options = {
-            proxy: ENV['http_proxy'] || ENV['https_proxy'], 
+            proxy: ENV['http_proxy'] || ENV['https_proxy'],
             proxyuserpwd: "#{ENV['HTTP_PROXY_USER']}:#{ENV['HTTP_PROXY_PASSWORD']}"
           }
-          
+
           if proxy_options[:proxy]
             request = Typhoeus::Request.new(
               upload_url,
               method: :put,
               proxy: proxy_options[:proxy],
               proxyuserpwd: proxy_options[:proxyuserpwd],
-              :body => put_data,
+              :body => body_data,
               :headers => {
                 'Content-Type' => 'application/octet-stream',
                 'token' => @token['token'],
@@ -182,48 +279,25 @@ module Pindo
             request = Typhoeus::Request.new(
               upload_url,
               method: :put,
-              :body => put_data,
+              :body => body_data,
               :headers => {
                 'Content-Type' => 'application/octet-stream',
                 'token' => @token['token'],
                 'Content-Length' => read_length.to_s
               }
             )
-          end
-
-
-          @upload_num = start_position
-          request.on_progress do |dltotal, dlnow, ultotal, ulnow|
-              # puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})"
-              if ulnow
-                @upload_num = start_position + ulnow
-                set_progress(@upload_num, filesize)
-              end
-          end
 
-          return_data = nil
-          request.on_complete do |response|
-              if response.success?
-                # puts "on_complete success"
-                return_data = response.headers["ETag"]
-              else
-                # puts "on_complete fail"
-              end
           end
-
-          request.run
-
-          return  return_data
-
+          return request
       end
 
-      def post_upload_finish_req(upload_key:nil, upload_id:nil, eTags:nil)
+      def post_upload_finish_req(upload_path_key:nil, upload_id:nil, eTags:nil)
 
           boss_url = @baseurl + @request_config["multi_signed_url_upload"]
 
           body_params = {
               functionName:"finish",
-              fileKey:upload_key,
+              fileKey:upload_path_key,
               uploadId:upload_id,
               tags:eTags
           }
@@ -247,16 +321,14 @@ module Pindo
       end
 
 
-        def post_upload_url_req(upload_key:nil, file_size:nil)
+        def post_upload_url_req(upload_path_key:nil, file_ceil_size:nil)
 
-          # puts "post_upload_url_req start ..."
-          # puts "file_size: #{file_size} MB"
           boss_url = @baseurl + @request_config["multi_signed_url_upload"]
 
           body_params = {
               functionName:"start",
-              fileKey:upload_key,
-              fileSize:file_size
+              fileKey:upload_path_key,
+              fileSize:file_ceil_size
           }
 
           con = HttpClient.create_instance_with_proxy
@@ -273,19 +345,71 @@ module Pindo
               result_data = JSON.parse(res.body)
           end
 
-          # puts JSON.pretty_generate(result_data)
-
           return  result_data
 
       end
 
-      def set_progress(index_num, total_num,  char ='>' )
-          progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
-          total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
-          index = 40.0 * index_num / total_num
-          upload_message = "已上传:#{progress_str}\%【" + (char * (index/1).floor).ljust(40.0, '_') +  "】Total: #{total_size} M"
-          Funlog.instance.fancyinfo_update(upload_message)
+      class PgyerUploadProgressBar
+
+        attr_accessor :draw_char
+        attr_accessor :complete_size
+        attr_accessor :upload_total_size
+        attr_accessor :last_update_time
+        attr_accessor :update_ing_size
+
+        def initialize(upload_total_size:nil, draw_char:'>')
+
+
+          @upload_total_size = upload_total_size
+          @draw_char = draw_char
+          @last_update_time = (Time.now.to_f * 1000).to_i #毫秒
+
+          @complete_size = 0
+          @update_ing_size = {}
+        end
+
+        def update_upload_index(upload_part:nil, upload_size:nil)
+            @update_ing_size[upload_part] = upload_size
+        end
+
+        def delete_upload_index(upload_part:nil)
+            @update_ing_size[upload_part] = 0
+        end
+
+        def complete_upload_index(upload_part:nil, complete_size:nil)
+            @complete_size = @complete_size + complete_size
+            @update_ing_size[upload_part] = 0
+        end
+
+        def update_upload_progress()
+            time_now = (Time.now.to_f * 1000).to_i #毫秒
+            if time_now - @last_update_time > 80
+                @last_update_time = time_now
+                total_num = @upload_total_size
+                index_num = @complete_size
+                @update_ing_size.each do |key, value|
+                    index_num = index_num + value
+                end
+
+                progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
+                total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
+                upload_size = sprintf("%.2f", 1.00 * index_num / 1024 /1024 )
+                index = 40.0 * index_num / total_num
+                upload_message = "已上传:#{upload_size}MB|#{progress_str}\%【" + (@draw_char * (index/1).floor).ljust(40.0, '_') +  "】Total:#{total_size}MB"
+                Funlog.instance.fancyinfo_update(upload_message)
+            end
+        end
+
       end
 
+      # def set_progress(index_num, total_num,  char ='>' )
+      #     progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
+      #     total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
+      #     upload_size = sprintf("%.2f", 1.00 * index_num / 1024 /1024 )
+      #     index = 40.0 * index_num / total_num
+      #     upload_message = "已上传:#{upload_size}MB|#{progress_str}\%【" + (char * (index/1).floor).ljust(40.0, '_') +  "】Total:#{total_size}MB"
+      #     Funlog.instance.fancyinfo_update(upload_message)
+      # end
+
   end
 end

data/lib/pindo/version.rb

@@ -1,6 +1,6 @@
 module Pindo
 
-  	VERSION = "4.7.7"
+  	VERSION = "4.7.8"
 
   	class VersionCheck
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pindo
 version: !ruby/object:Gem::Version
-  version: 4.7.7
+  version: 4.7.8
 platform: ruby
 authors:
 - wade
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-22 00:00:00.000000000 Z
+date: 2024-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: claide