jpsclient 0.2.0

data/lib/jpsclient/upload/upload_client.rb

@@ -0,0 +1,334 @@
+require 'securerandom'
+require 'typhoeus'
+require 'thread'
+require 'etc'
+require 'digest'
+require 'jpsclient/base/exception'
+require 'jpsclient/upload/upload_config'
+require 'jpsclient/upload/upload_progress'
+require 'jpsclient/utils/logger'
+
+module JPSClient
+
+
+  class UploadClient
+
+      attr_accessor :upload_binary_file
+      attr_accessor :file_size
+      attr_accessor :progress_bar
+      attr_reader :jps_client
+
+      def initialize(jps_client)
+          raise ExceptionError, "必须提供 Client 实例" unless jps_client
+
+          @jps_client = jps_client
+
+          # 从 Client 获取所需资源
+          config_json = @jps_client.config_json
+
+          # 加载上传配置
+          @upload_config = UploadConfig.from_json(config_json["upload_config"])
+          unless @upload_config && @upload_config.valid?
+            raise ExceptionError, "上传配置无效或不完整"
+          end
+          # 添加互斥锁用于线程安全
+          @upload_eTags_mutex = Mutex.new
+          @tasks_queue_mutex = Mutex.new
+          @active_tasks_mutex = Mutex.new
+          @upload_failed_mutex = Mutex.new
+          @upload_failed = false
+      end
+
+      def upload_file(binary_file:nil, isAttach:false)
+
+        raise ExceptionError, "上传文件不能为空" if binary_file.nil? || !File.exist?(binary_file)
+
+        @upload_binary_file = binary_file
+        @file_size = File.size(@upload_binary_file)
+        @progress_bar = UploadProgress.new(upload_total_size:@file_size)
+        @upload_failed = false  # 重置上传失败标志
+
+        # 生成S3 Key (保持UUID方式)
+        file_uuid = SecureRandom.uuid
+        extension = File.extname(@upload_binary_file)
+        filename = File.basename(@upload_binary_file)
+
+        # 使用配置中的路径前缀
+        path_prefix = isAttach ? @upload_config.attach_url : @upload_config.default_url
+        s3_key = "#{path_prefix}#{file_uuid}#{extension}"
+
+        # 计算文件大小和分片信息
+        total_mb = sprintf("%.2f", @file_size / 1024.0 / 1024.0)
+        chunk_size = @upload_config.chunk_size_bytes
+        chunks = (@file_size.to_f / chunk_size).ceil
+
+        # 使用配置的并发工作线程数和重试次数
+        task_num = @upload_config.concurrent_workers
+        retry_count = @upload_config.max_retry_times
+
+        puts "文件路径: #{@upload_binary_file}"
+        puts "文件大小: #{total_mb}M"
+        puts "上传路径: #{s3_key}"
+        puts
+        puts "切片大小: #{@upload_config.chunk_size_mb}MB"
+
+        upload_result = nil
+
+        begin
+          # 步骤1: 初始化多部分上传
+          Logger.instance.fancyinfo_start("初始化上传...")
+          init_result = @jps_client.init_file_multipart(
+              s3_key: s3_key,
+              content_type: "",
+              upload_config: @upload_config
+          )
+
+          if init_result.nil? || !init_result.dig("data", "upload_id")
+              raise ExceptionError, "初始化上传失败，请检查网络或服务器状态"
+          end
+
+          upload_id = init_result["data"]["upload_id"]
+
+          # 准备分片任务队列
+          @tasks_queue = Queue.new
+          @worker_threads = []
+          @expected_parts = chunks  # 保存预期的分片数量
+
+          # 创建分片任务
+          chunks.times do |i|
+              task_item = {
+                  "partNo" => i + 1,
+                  "s3Key" => s3_key,
+                  "uploadId" => upload_id,
+                  "retryCount" => retry_count
+              }
+              @tasks_queue.push(task_item)
+          end
+
+          puts "文件分片数: #{chunks}"
+          puts "并发上传线程数: #{task_num} (CPU核心数: #{Etc.nprocessors})"
+          puts "失败重试次数: #{retry_count}"
+          puts
+
+          Logger.instance.fancyinfo_start("开始上传...")
+          @upload_eTags = []
+          @active_tasks = 0  # 跟踪活动任务数量
+
+          continuous_upload_data_req(concurrency: task_num)
+
+          # 检查上传是否全部成功
+          if upload_failed? || @upload_eTags.length != @expected_parts
+              upload_result = nil
+              Logger.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败! 😭😭😭")
+              return upload_result
+          end
+
+          # 步骤3: 完成多部分上传
+          Logger.instance.fancyinfo_start("完成上传...")
+          complete_result = @jps_client.complete_file_multipart(
+              s3_key: s3_key,
+              upload_id: upload_id,
+              upload_config: @upload_config
+          )
+
+          if complete_result && complete_result["code"] == 200
+              upload_result = complete_result.dig("data", "url") || s3_key
+              Logger.instance.fancyinfo_success("文件#{@upload_binary_file} 上传成功! 😎😎😎")
+          else
+              upload_result = nil
+              error_msg = complete_result["msg"] || complete_result["message"] || "未知错误"
+              Logger.instance.fancyinfo_error("文件#{@upload_binary_file} 上传失败: #{error_msg} 😭😭😭")
+          end
+
+        rescue => e
+          upload_result = nil
+          Logger.instance.fancyinfo_error("文件上传过程发生异常: #{e.message} 😭😭😭")
+        ensure
+          # 确保所有工作线程都被清理
+          cleanup_worker_threads
+        end
+
+        return upload_result
+
+      end
+
+      # 安全地检查上传失败状态
+      def upload_failed?
+        @upload_failed_mutex.synchronize { @upload_failed }
+      end
+
+      # 安全地设置上传失败状态
+      def set_upload_failed(error_msg = nil)
+        @upload_failed_mutex.synchronize do
+          @upload_failed = true
+          Logger.instance.fancyinfo_error("上传失败: #{error_msg}") if error_msg
+        end
+      end
+
+      # 清理所有工作线程
+      def cleanup_worker_threads
+        @worker_threads.each do |thread|
+          # 尝试安全终止线程
+          thread.exit if thread.alive?
+        end
+        @worker_threads.clear
+      end
+
+      def continuous_upload_data_req(concurrency:1)
+          # 使用固定大小的线程池，避免线程无限增长
+          @worker_threads = []
+          @active_tasks = 0
+          @task_complete_cv = ConditionVariable.new
+
+          # 创建固定数量的工作线程
+          concurrency.times do
+              @worker_threads << Thread.new { worker_loop }
+          end
+
+          # 设置超时保护
+          timeout_seconds = 300  # 5分钟超时
+          start_time = Time.now
+
+          # 主线程等待所有任务完成
+          @tasks_queue_mutex.synchronize do
+              while (@active_tasks > 0 || !@tasks_queue.empty?) && !upload_failed?
+                  remaining_time = timeout_seconds - (Time.now - start_time)
+                  if remaining_time <= 0
+                    set_upload_failed("上传任务超时")
+                    break
+                  end
+
+                  # 等待任务完成通知
+                  @task_complete_cv.wait(@tasks_queue_mutex, [remaining_time, 30].min)
+              end
+          end
+
+          # 停止所有工作线程
+          @worker_threads.each { |t| t.kill }
+
+          # 检查所有分片是否都上传成功
+          if @upload_eTags.length != @expected_parts && !upload_failed?
+              set_upload_failed("部分分片上传失败，已上传#{@upload_eTags.length}/#{@expected_parts}")
+          end
+      end
+
+      # 工作线程的主循环
+      def worker_loop
+          loop do
+              upload_params_item = nil
+
+              # 从队列获取任务
+              @tasks_queue_mutex.synchronize do
+                  return if upload_failed? && @tasks_queue.empty?
+
+                  if @tasks_queue.empty?
+                      # 队列为空，等待新任务
+                      @task_complete_cv.wait(@tasks_queue_mutex, 1)
+                      next
+                  end
+
+                  upload_params_item = @tasks_queue.pop
+                  @active_tasks_mutex.synchronize { @active_tasks += 1 }
+              end
+
+              # 处理任务
+              if upload_params_item
+                  begin
+                      process_upload_task(upload_params_item)
+                  rescue => e
+                      set_upload_failed("处理分片#{upload_params_item["partNo"]}时出错: #{e.message}")
+                  ensure
+                      @active_tasks_mutex.synchronize { @active_tasks -= 1 }
+                      @tasks_queue_mutex.synchronize { @task_complete_cv.broadcast }
+                  end
+              end
+
+              # 检查是否应该退出
+              break if upload_failed?
+          end
+      end
+
+      def process_upload_task(upload_params_item)
+          part_no = upload_params_item["partNo"]
+          s3_key = upload_params_item["s3Key"]
+          upload_id = upload_params_item["uploadId"]
+
+          # 步骤2: 获取分片的预签名URL
+          sign_result = @jps_client.get_file_sign_url(
+              s3_key: s3_key,
+              upload_id: upload_id,
+              part_id: part_no,
+              upload_config: @upload_config
+          )
+
+          if sign_result.nil? || !sign_result.dig("data", "url")
+              raise ExceptionError, "获取分片#{part_no}的上传URL失败"
+          end
+
+          upload_url = sign_result["data"]["url"]
+
+          # 计算分片数据范围（使用配置的分片大小）
+          chunk_size = @upload_config.chunk_size_bytes
+          start_position = chunk_size * (part_no - 1)
+          if part_no * chunk_size > @file_size
+              read_length = @file_size - start_position
+          else
+              read_length = chunk_size
+          end
+
+          file = File.open(@upload_binary_file, "rb")
+          begin
+              file.seek(start_position)
+              put_data = file.read(read_length)
+
+              # 创建上传请求（直接使用预签名URL）
+              request = Typhoeus::Request.new(
+                  upload_url,
+                  method: :put,
+                  body: put_data,
+                  headers: {
+                      'Content-Type' => 'application/octet-stream',
+                      'Content-Length' => read_length.to_s
+                  },
+                  timeout: 300  # 5分钟超时
+              )
+
+              # 设置上传进度回调
+              upload_size_last = 0
+              request.on_progress do |dltotal, dlnow, ultotal, ulnow|
+                  if ulnow && ulnow > upload_size_last
+                      upload_size_last = ulnow
+                      @progress_bar.update_upload_index(upload_part:part_no, upload_size:ulnow)
+                      @progress_bar.update_upload_progress()
+                  end
+              end
+
+              # 设置请求超时
+              request.options[:timeout] = 300  # 5分钟超时
+
+              # 执行请求并等待完成
+              response = request.run
+
+              # 处理响应结果
+              if response.success?
+                  @progress_bar.complete_upload_index(upload_part:part_no, complete_size:read_length)
+                  # 新API不再需要收集ETag，只记录成功的分片号
+                  @upload_eTags_mutex.synchronize { @upload_eTags << part_no }
+              else
+                  @progress_bar.delete_upload_index(upload_part:part_no)
+                  upload_params_item["retryCount"] = upload_params_item["retryCount"] - 1
+                  if upload_params_item["retryCount"] > 0
+                      # 重试任务
+                      @tasks_queue_mutex.synchronize { @tasks_queue.push(upload_params_item) }
+                  else
+                      set_upload_failed("文件#{@upload_binary_file} 分片#{part_no}上传失败: HTTP #{response.code}")
+                  end
+              end
+          ensure
+              file.close
+          end
+      end
+
+
+  end
+end

data/lib/jpsclient/upload/upload_config.rb

@@ -0,0 +1,128 @@
+require 'etc'
+
+module JPSClient
+  # 上传配置类
+  class UploadConfig
+    attr_accessor :region
+    attr_accessor :bucket_name
+    attr_accessor :access_key_id
+    attr_accessor :access_key_secret
+    attr_accessor :default_url
+    attr_accessor :attach_url
+    attr_accessor :upload_type         # 新增：上传类型
+    attr_accessor :concurrent_workers   # 新增：并发上传工作线程数
+    attr_accessor :chunk_size_mb       # 新增：分片大小（MB）
+    attr_accessor :max_retry_times     # 新增：最大重试次数
+
+    def initialize(
+      region: nil,
+      bucket_name: nil,
+      access_key_id: nil,
+      access_key_secret: nil,
+      default_url: nil,
+      attach_url: nil,
+      upload_type: nil,
+      concurrent_workers: nil,
+      chunk_size_mb: nil,
+      max_retry_times: nil
+    )
+      @region = region
+      @bucket_name = bucket_name
+      @access_key_id = access_key_id
+      @access_key_secret = access_key_secret
+      @default_url = default_url
+      @attach_url = attach_url
+      @upload_type = upload_type
+      @concurrent_workers = concurrent_workers
+      @chunk_size_mb = chunk_size_mb
+      @max_retry_times = max_retry_times
+    end
+
+    # 从 JSON 配置创建实例
+    def self.from_json(json_config)
+      return nil unless json_config
+
+      # 动态计算并发数
+      concurrent_workers = determine_concurrent_workers(json_config['concurrent_workers'])
+
+      new(
+        region: json_config['region'] || "ap-southeast-1",
+        bucket_name: json_config['bucket_name'] || "pgy-resource-new",
+        access_key_id: json_config['access_key_id'],
+        access_key_secret: json_config['access_key_secret'],
+        default_url: json_config['default_url'] || "resource/",
+        attach_url: json_config['attach_url'] || "attach_file/",
+        upload_type: json_config['upload_type'] || "s3",
+        concurrent_workers: concurrent_workers,
+        chunk_size_mb: json_config['chunk_size_mb'] || 5,
+        max_retry_times: json_config['max_retry_times'] || 3
+      )
+    end
+
+    # 智能确定并发工作线程数
+    # 策略：使用 min(CPU自适应值, 配置最大值)
+    def self.determine_concurrent_workers(config_value)
+      cpu_count = Etc.nprocessors
+
+      # 根据CPU计算推荐值（I/O密集型任务，使用CPU核心数的2倍）
+      cpu_recommended = cpu_count * 2
+
+      # 处理不同的配置值
+      if config_value.nil?
+        # 没有配置，使用CPU推荐值，但限制在4-16之间
+        return [[cpu_recommended, 4].max, 16].min
+      elsif config_value.to_s.downcase == 'auto'
+        # 自动模式：使用CPU推荐值，限制在4-16之间
+        return [[cpu_recommended, 4].max, 16].min
+      elsif config_value.is_a?(String) && config_value.include?('cpu')
+        # 支持 "cpu*2", "cpu*3" 这样的配置
+        if config_value =~ /cpu\s*\*\s*(\d+)/
+          multiplier = $1.to_i
+          calculated = cpu_count * multiplier
+          # 限制在 2-30 之间
+          return [[calculated, 2].max, 30].min
+        end
+      end
+
+      # 配置值是数字时，作为最大限制
+      max_workers = config_value.to_i
+      if max_workers > 0
+        # 实际使用：min(CPU推荐值, 配置最大值)
+        # 但至少保证2个线程
+        actual_workers = [cpu_recommended, max_workers].min
+        return [actual_workers, 2].max
+      end
+
+      # 默认值
+      return 5
+    end
+
+    # 验证配置完整性
+    def valid?
+      !@region.nil? && !@bucket_name.nil? &&
+      !@access_key_id.nil? && !@access_key_secret.nil? &&
+      !@default_url.nil? && !@attach_url.nil?
+    end
+
+    # 获取分片大小（字节）
+    def chunk_size_bytes
+      @chunk_size_mb * 1024 * 1024
+    end
+
+    # 转换为 Hash
+    def to_h
+      {
+        'region' => @region,
+        'bucket_name' => @bucket_name,
+        'access_key_id' => @access_key_id,
+        'access_key_secret' => @access_key_secret,
+        'default_url' => @default_url,
+        'attach_url' => @attach_url,
+        'upload_type' => @upload_type,
+        'concurrent_workers' => @concurrent_workers,
+        'chunk_size_mb' => @chunk_size_mb,
+        'max_retry_times' => @max_retry_times
+      }
+    end
+  end
+end

data/lib/jpsclient/upload/upload_progress.rb

@@ -0,0 +1,73 @@
+require 'thread'
+require 'jpsclient/utils/logger'
+
+module JPSClient
+  # 上传进度条管理类
+  class UploadProgress
+
+    attr_accessor :draw_char
+    attr_accessor :complete_size
+    attr_accessor :upload_total_size
+    attr_accessor :last_update_time
+    attr_accessor :update_ing_size
+    attr_accessor :is_done
+
+    def initialize(upload_total_size:nil, draw_char:'>')
+      @upload_total_size = upload_total_size
+      @draw_char = draw_char
+      @last_update_time = (Time.now.to_f * 1000).to_i #毫秒
+
+      @complete_size = 0
+      @update_ing_size = {}
+      @is_done = false
+
+      # 添加互斥锁来保护进度条更新
+      @mutex = Mutex.new
+    end
+
+    def update_upload_index(upload_part:nil, upload_size:nil)
+        @mutex.synchronize do
+            @update_ing_size[upload_part] = upload_size
+        end
+    end
+
+    def delete_upload_index(upload_part:nil)
+        @mutex.synchronize do
+            @update_ing_size[upload_part] = 0
+        end
+    end
+
+    def complete_upload_index(upload_part:nil, complete_size:nil)
+        @mutex.synchronize do
+            @complete_size = @complete_size + complete_size
+            @update_ing_size[upload_part] = 0
+        end
+    end
+
+    def update_upload_progress()
+        time_now = (Time.now.to_f * 1000).to_i #毫秒
+        if time_now - @last_update_time > 80
+            @mutex.synchronize do
+                @last_update_time = time_now
+                total_num = @upload_total_size
+                index_num = @complete_size
+                @update_ing_size.each do |key, value|
+                   index_num = index_num + value
+                end
+
+                progress_str = sprintf("%.2f", 100.0 * index_num / total_num )
+                total_size = sprintf("%.2f", 1.00 * total_num / 1024 /1024 )
+                upload_size = sprintf("%.2f", 1.00 * index_num / 1024 /1024 )
+                index = 40.0 * index_num / total_num
+                upload_message = "已上传:#{upload_size}MB|#{progress_str}\%【" + (@draw_char * (index/1).floor).ljust(40.0, '_') +  "】Total:#{total_size}MB"
+                Logger.instance.fancyinfo_update(upload_message)
+                if index_num == total_num && !@is_done
+                  @is_done = true
+                  Logger.instance.fancyinfo_success(upload_message)
+                end
+            end
+        end
+    end
+
+  end
+end

data/lib/jpsclient/utils/aes.rb

@@ -0,0 +1,49 @@
+require 'openssl'
+require 'base64'
+
+module JPSClient
+  # 独立的 AES 加密解密类
+  class AES
+    CIPHER = 'AES-128-ECB'
+
+    def initialize(key)
+      @key = key
+      validate_key!
+    end
+
+    # 加密字符串
+    def encrypt(plain_text)
+      return nil if plain_text.nil? || plain_text.empty?
+
+      cipher = OpenSSL::Cipher.new(CIPHER)
+      cipher.encrypt
+      cipher.key = @key
+
+      encrypted = cipher.update(plain_text) + cipher.final
+      Base64.strict_encode64(encrypted)
+    rescue => e
+      raise ExceptionError, "加密失败: #{e.message}"
+    end
+
+    # 解密字符串
+    def decrypt(encrypted_text)
+      return nil if encrypted_text.nil? || encrypted_text.empty?
+
+      cipher = OpenSSL::Cipher.new(CIPHER)
+      cipher.decrypt
+      cipher.key = @key
+
+      decoded = Base64.strict_decode64(encrypted_text)
+      cipher.update(decoded) + cipher.final
+    rescue => e
+      raise ExceptionError, "解密失败: #{e.message}"
+    end
+
+    private
+
+    def validate_key!
+      raise ExceptionError, "AES密钥不能为空" if @key.nil? || @key.empty?
+      raise ExceptionError, "AES密钥长度必须为16字节" if @key.bytesize != 16
+    end
+  end
+end

data/lib/jpsclient/utils/logger.rb

@@ -0,0 +1,38 @@
+module JPSClient
+  # 简单的Logger实现，替代Funlog
+  class Logger
+    def self.instance
+      @instance ||= new
+    end
+
+    def initialize
+      @verbose = ENV['JPS_DEBUG'] == 'true'
+    end
+
+    def fancyinfo_start(message)
+      puts "▶ #{message}" if @verbose || true  # 总是显示开始信息
+    end
+
+    def fancyinfo_success(message)
+      puts "✅ #{message}"
+    end
+
+    def fancyinfo_error(message)
+      puts "❌ #{message}"
+    end
+
+    def fancyinfo_update(message)
+      # 进度更新，使用 \r 实现覆盖
+      print "\r#{message}"
+      $stdout.flush
+    end
+
+    def info(message)
+      puts message if @verbose
+    end
+
+    def error(message)
+      $stderr.puts "ERROR: #{message}"
+    end
+  end
+end

data/lib/jpsclient/version.rb

@@ -0,0 +1,3 @@
+module JPSClient
+  VERSION = "0.2.0"
+end

data/lib/jpsclient.rb

@@ -0,0 +1,34 @@
+# 主入口文件
+require 'jpsclient/version'
+
+# 基础模块
+require 'jpsclient/base/exception'
+require 'jpsclient/base/api_config'
+
+# API 客户端
+require 'jpsclient/api/client'
+
+# 工具模块
+require 'jpsclient/utils/logger'
+require 'jpsclient/utils/aes'
+
+# HTTP 模块
+require 'jpsclient/http/http_client'
+
+# 认证模块
+require 'jpsclient/auth/token'
+require 'jpsclient/auth/auth'
+
+# 上传模块
+require 'jpsclient/upload/upload_config'
+require 'jpsclient/upload/upload_progress'
+require 'jpsclient/upload/upload_client'
+
+module JPSClient
+  class << self
+    # 便捷方法：创建客户端实例
+    def new(config_file:)
+      Client.new(config_file: config_file)
+    end
+  end
+end