jpsclient 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f185bd85f53ebc4262ecc570b1bb0d69ace35f375f063f2b536109fc988f38d5
-  data.tar.gz: 3c34fde3e18d9ae5f98822f5c629fb1c693dbc54afc572f6db871c467ad322d8
+  metadata.gz: 119466770ec55de57abefc0e70ea4aa8ecd481f03e8d1cc949c476c183c0b128
+  data.tar.gz: 4b86a7ce21d27547e244fb74d0deaaecefb4b8f30d90bbba27b7a82391314b1b
 SHA512:
-  metadata.gz: dcd4b34b4f6171933976af6d3cf13bb712245abcefb74359b29f2c1862ceb95c7ef0398db86b2b73cb78b3537b2511c9f52a63e282dbc2334d0179c4547de8db
-  data.tar.gz: 156f2d4b7940a58e9936bdcac1d91b3d1b93a52f84768b68d38c5c32a5459836a8f67b046d07f12261d155527cf9974673332d21df97b5c3c25cc762e198ea1f
+  metadata.gz: bc4f5f634d54c043bf6e113af55678fd343babca1ad6a903c9e565c33f47fa333ed503741f23fe803e5b15882738804256d23177e21a2a424916e98156a794b5
+  data.tar.gz: d8b96409527709af81761bab9d69b92d433b6b744c13474cb185a8462566327480c4cc6a9b923f8cb441d88bd4df4a4bfcf543eb3708959cf72eb0045bc324fd

data/lib/jpsclient/upload/upload_client.rb

@@ -45,6 +45,12 @@ module JPSClient
 
         @upload_binary_file = binary_file
         @file_size = File.size(@upload_binary_file)
+
+        # 处理空文件
+        if @file_size == 0
+            raise ExceptionError, "不能上传空文件: #{binary_file}"
+        end
+
         @progress_bar = UploadProgress.new(upload_total_size:@file_size)
         @upload_failed = false  # 重置上传失败标志
 
@@ -134,11 +140,12 @@ module JPSClient
           )
 
           if complete_result && complete_result["code"] == 200
+              # 返回 URL 字符串以保持向后兼容
               upload_result = complete_result.dig("data", "url") || s3_key
               Logger.instance.fancyinfo_success("文件#{@upload_binary_file} 上传成功! 😎😎😎")
           else
               upload_result = nil
-              error_msg = complete_result["msg"] || complete_result["message"] || "未知错误"
+              error_msg = complete_result ? (complete_result["msg"] || complete_result["message"] || "未知错误") : "未知错误"
               Logger.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败: #{error_msg} 😭😭😭")
           end
 
@@ -169,6 +176,8 @@ module JPSClient
 
       # 清理所有工作线程
       def cleanup_worker_threads
+        return unless @worker_threads  # 防止未初始化
+
         @worker_threads.each do |thread|
           # 尝试安全终止线程
           thread.exit if thread.alive?
@@ -180,6 +189,7 @@ module JPSClient
           # 使用固定大小的线程池，避免线程无限增长
           @worker_threads = []
           @active_tasks = 0
+          @stop_workers = false
           @task_complete_cv = ConditionVariable.new
 
           # 创建固定数量的工作线程
@@ -187,29 +197,31 @@ module JPSClient
               @worker_threads << Thread.new { worker_loop }
           end
 
-          # 设置超时保护
-          timeout_seconds = 300  # 5分钟超时
+          # 主线程等待所有任务完成
+          # 没有总体超时限制，会一直等待直到：
+          # 1. 所有分片上传成功
+          # 2. 某个分片在重试后彻底失败
           start_time = Time.now
 
-          # 主线程等待所有任务完成
           @tasks_queue_mutex.synchronize do
               while (@active_tasks > 0 || !@tasks_queue.empty?) && !upload_failed?
-                  remaining_time = timeout_seconds - (Time.now - start_time)
-                  if remaining_time <= 0
-                    set_upload_failed("上传任务超时")
-                    break
-                  end
-
-                  # 等待任务完成通知
-                  @task_complete_cv.wait(@tasks_queue_mutex, [remaining_time, 30].min)
+                  # 等待任务完成通知，每30秒唤醒一次检查状态
+                  @task_complete_cv.wait(@tasks_queue_mutex, 30)
               end
           end
 
           # 停止所有工作线程
-          @worker_threads.each { |t| t.kill }
+          @stop_workers = true
+          @tasks_queue_mutex.synchronize { @task_complete_cv.broadcast }
 
-          # 检查所有分片是否都上传成功
-          if @upload_eTags.length != @expected_parts && !upload_failed?
+          # 等待所有线程结束（最多等待5秒）
+          @worker_threads.each do |t|
+              t.join(5)
+              t.kill if t.alive?  # 如果仍然存活，强制终止
+          end
+
+          # 只在非失败情况下检查分片完整性
+          if !upload_failed? && @upload_eTags.length != @expected_parts
               set_upload_failed("部分分片上传失败，已上传#{@upload_eTags.length}/#{@expected_parts}")
           end
       end
@@ -221,7 +233,7 @@ module JPSClient
 
               # 从队列获取任务
               @tasks_queue_mutex.synchronize do
-                  return if upload_failed? && @tasks_queue.empty?
+                  return if @stop_workers || (upload_failed? && @tasks_queue.empty?)
 
                   if @tasks_queue.empty?
                       # 队列为空，等待新任务
@@ -230,9 +242,11 @@ module JPSClient
                   end
 
                   upload_params_item = @tasks_queue.pop
-                  @active_tasks_mutex.synchronize { @active_tasks += 1 }
               end
 
+              # 在锁外增加活跃任务计数，避免嵌套锁
+              @active_tasks_mutex.synchronize { @active_tasks += 1 } if upload_params_item
+
               # 处理任务
               if upload_params_item
                   begin
@@ -254,6 +268,14 @@ module JPSClient
           part_no = upload_params_item["partNo"]
           s3_key = upload_params_item["s3Key"]
           upload_id = upload_params_item["uploadId"]
+          retry_count = upload_params_item["retryCount"]
+
+          # 记录重试信息（仅在重试时记录）
+          # 注释掉以避免打断进度条
+          # current_attempt = @upload_config.max_retry_times - retry_count + 1
+          # if current_attempt > 1
+          #     Logger.instance.info("上传分片 ##{part_no}，第 #{current_attempt} 次尝试（共 #{@upload_config.max_retry_times + 1} 次机会）")
+          # end
 
           # 步骤2: 获取分片的预签名URL
           sign_result = @jps_client.get_file_sign_url(
@@ -264,7 +286,9 @@ module JPSClient
           )
 
           if sign_result.nil? || !sign_result.dig("data", "url")
-              raise ExceptionError, "获取分片#{part_no}的上传URL失败"
+              # Logger.instance.info("分片 ##{part_no} 获取上传URL失败")  # 避免打断进度条
+              handle_retry(upload_params_item, "获取上传URL失败")
+              return
           end
 
           upload_url = sign_result["data"]["url"]
@@ -278,63 +302,106 @@ module JPSClient
               read_length = chunk_size
           end
 
-          file = File.open(@upload_binary_file, "rb")
+          # 读取文件数据（每个线程独立打开文件，无需互斥锁）
+          put_data = nil
           begin
-              file.seek(start_position)
-              put_data = file.read(read_length)
-
-              # 创建上传请求（直接使用预签名URL）
-              request = Typhoeus::Request.new(
-                  upload_url,
-                  method: :put,
-                  body: put_data,
-                  headers: {
-                      'Content-Type' => 'application/octet-stream',
-                      'Content-Length' => read_length.to_s
-                  },
-                  timeout: 300  # 5分钟超时
-              )
-
-              # 设置上传进度回调
-              upload_size_last = 0
-              request.on_progress do |dltotal, dlnow, ultotal, ulnow|
-                  if ulnow && ulnow > upload_size_last
-                      upload_size_last = ulnow
+              file = File.open(@upload_binary_file, "rb")
+              begin
+                  file.seek(start_position)
+                  put_data = file.read(read_length)
+              ensure
+                  file.close if file
+              end
+          rescue => e
+              # Logger.instance.info("分片 ##{part_no} 读取文件失败: #{e.message}")  # 避免打断进度条
+              handle_retry(upload_params_item, "文件读取失败: #{e.message}")
+              return
+          end
+
+          # 创建上传请求（直接使用预签名URL）
+          # 对于单个分片，使用更合理的超时时间（根据分片大小动态调整）
+          # 5MB 分片在慢速网络(100KB/s)下需要约50秒，所以基础超时应该更长
+          chunk_timeout = calculate_chunk_timeout(read_length)
+
+          request = Typhoeus::Request.new(
+              upload_url,
+              method: :put,
+              body: put_data,
+              headers: {
+                  'Content-Type' => 'application/octet-stream',
+                  'Content-Length' => read_length.to_s
+              },
+              timeout: chunk_timeout,
+              connecttimeout: 30  # 连接超时30秒
+          )
+
+          # 设置上传进度回调
+          upload_size_last = 0
+          last_progress_time = Time.now
+          request.on_progress do |dltotal, dlnow, ultotal, ulnow|
+              if ulnow && ulnow > upload_size_last
+                  upload_size_last = ulnow
+                  # 限制进度更新频率，避免过于频繁的更新
+                  if Time.now - last_progress_time > 0.5  # 每0.5秒更新一次
                       @progress_bar.update_upload_index(upload_part:part_no, upload_size:ulnow)
                       @progress_bar.update_upload_progress()
+                      last_progress_time = Time.now
                   end
               end
+          end
 
-              # 设置请求超时
-              request.options[:timeout] = 300  # 5分钟超时
-
-              # 执行请求并等待完成
-              response = request.run
+          # 执行请求并等待完成
+          response = request.run
 
-              # 处理响应结果
-              if response.success?
-                  @progress_bar.complete_upload_index(upload_part:part_no, complete_size:read_length)
-                  # 只有在未完成时才更新进度条，避免100%时重复显示
-                  unless @progress_bar.is_done
-                    @progress_bar.update_upload_progress()
-                  end
-                  # 新API不再需要收集ETag，只记录成功的分片号
-                  @upload_eTags_mutex.synchronize { @upload_eTags << part_no }
-              else
-                  @progress_bar.delete_upload_index(upload_part:part_no)
-                  upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
-                  if upload_params_item["retryCount"] > 0
-                      # 重试任务
-                      @tasks_queue_mutex.synchronize { @tasks_queue.push(upload_params_item) }
-                  else
-                      set_upload_failed("文件#{@upload_binary_file} 分片#{part_no}上传失败: HTTP #{response.code}")
-                  end
+          # 处理响应结果
+          if response.success?
+              @progress_bar.complete_upload_index(upload_part:part_no, complete_size:read_length)
+              # 只有在未完成时才更新进度条，避免100%时重复显示
+              unless @progress_bar.is_done
+                @progress_bar.update_upload_progress()
               end
-          ensure
-              file.close
+              # 新API不再需要收集ETag，只记录成功的分片号
+              @upload_eTags_mutex.synchronize { @upload_eTags << part_no }
+              # Logger.instance.info("分片 ##{part_no} 上传成功")  # 避免打断进度条
+          elsif response.timed_out?
+              # 超时处理
+              # Logger.instance.info("分片 ##{part_no} 上传超时，响应时间: #{response.time}秒")  # 避免打断进度条
+              handle_retry(upload_params_item, "超时")
+          elsif response.code == 0
+              # 网络错误
+              error_msg = response.return_message || "未知网络错误"
+              # Logger.instance.info("分片 ##{part_no} 网络错误: #{error_msg}")  # 避免打断进度条
+              handle_retry(upload_params_item, error_msg)
+          else
+              # HTTP错误
+              # Logger.instance.info("分片 ##{part_no} HTTP错误: #{response.code}, Body: #{response.body[0..200] if response.body}")  # 避免打断进度条
+              handle_retry(upload_params_item, "HTTP #{response.code}")
+          end
+      end
+
+      def calculate_chunk_timeout(chunk_size)
+          # 使用配置的 timeout_seconds 作为单个分片的超时时间
+          # 默认600秒（10分钟）
+          @upload_config.timeout_seconds || 600
+      end
+
+      def handle_retry(upload_params_item, error_reason)
+          part_no = upload_params_item["partNo"]
+          @progress_bar.delete_upload_index(upload_part:part_no)
+
+          upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
+          if upload_params_item["retryCount"] > 0
+              # 简单重试，直接放回队列（避免创建额外线程）
+              # Logger.instance.info("分片 ##{part_no} 准备重试，剩余重试次数: #{upload_params_item["retryCount"]}")  # 避免打断进度条
+
+              # 直接放回队列末尾，让其他分片先执行
+              @tasks_queue_mutex.synchronize { @tasks_queue.push(upload_params_item) }
+          else
+              set_upload_failed("文件#{@upload_binary_file} 分片#{part_no}上传失败: #{error_reason}，已达最大重试次数")
           end
       end
 
 
+
   end
 end

data/lib/jpsclient/upload/upload_config.rb

@@ -12,7 +12,8 @@ module JPSClient
     attr_accessor :upload_type         # 新增：上传类型
     attr_accessor :concurrent_workers   # 新增：并发上传工作线程数
     attr_accessor :chunk_size_mb       # 新增：分片大小（MB）
-    attr_accessor :max_retry_times     # 新增：最大重试次数
+    attr_accessor :max_retry_times     # 新增：每个分片的最大重试次数
+    attr_accessor :timeout_seconds     # 新增：单个分片的超时时间（秒）
 
     def initialize(
       region: nil,
@@ -24,7 +25,8 @@ module JPSClient
       upload_type: nil,
       concurrent_workers: nil,
       chunk_size_mb: nil,
-      max_retry_times: nil
+      max_retry_times: nil,
+      timeout_seconds: nil
     )
       @region = region
       @bucket_name = bucket_name
@@ -36,6 +38,7 @@ module JPSClient
       @concurrent_workers = concurrent_workers
       @chunk_size_mb = chunk_size_mb
       @max_retry_times = max_retry_times
+      @timeout_seconds = timeout_seconds
     end
 
     # 从 JSON 配置创建实例
@@ -55,7 +58,8 @@ module JPSClient
         upload_type: json_config['upload_type'] || "s3",
         concurrent_workers: concurrent_workers,
         chunk_size_mb: json_config['chunk_size_mb'] || 5,
-        max_retry_times: json_config['max_retry_times'] || 3
+        max_retry_times: json_config['max_retry_times'] || 6,  # 每个分片重试6次
+        timeout_seconds: json_config['timeout_seconds'] || 600  # 单个分片超时10分钟
       )
     end
 
@@ -121,7 +125,8 @@ module JPSClient
         'upload_type' => @upload_type,
         'concurrent_workers' => @concurrent_workers,
         'chunk_size_mb' => @chunk_size_mb,
-        'max_retry_times' => @max_retry_times
+        'max_retry_times' => @max_retry_times,
+        'timeout_seconds' => @timeout_seconds
       }
     end
   end

data/lib/jpsclient/version.rb

@@ -1,3 +1,3 @@
 module JPSClient
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: jpsclient
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Your Name
 bindir: bin
 cert_chain: []
-date: 2025-09-29 00:00:00.000000000 Z
+date: 2025-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday