kdeploy 1.2.41 → 1.3.3

data/lib/kdeploy/executor/command_executor.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+module Kdeploy
+  # Executes a single command and records execution time
+  class CommandExecutor
+    def initialize(executor, output, debug: false, retries: 0, retry_delay: 1, retry_on_nonzero: false,
+                   step_timeout: nil, retry_policy: nil)
+      @executor = executor
+      @output = output
+      @debug = debug
+      @retries = retries.to_i
+      @retry_delay = retry_delay.to_f
+      @retry_on_nonzero = retry_on_nonzero
+      @step_timeout = step_timeout&.to_f
+      @retry_policy = retry_policy
+    end
+
+    def execute_run(command, _host_name)
+      cmd = command[:command]
+      use_sudo = command[:sudo]
+
+      result, duration = measure_time do
+        with_retries(step_type: :run) do
+          with_timeout { @executor.execute(cmd, use_sudo: use_sudo) }
+        end
+      end
+
+      { command: cmd, output: result, duration: duration, type: :run }
+    end
+
+    def execute_upload(command, _host_name)
+      _result, duration = measure_time do
+        with_retries(step_type: :upload) do
+          with_timeout { @executor.upload(command[:source], command[:destination]) }
+        end
+      end
+      {
+        command: "upload: #{command[:source]} -> #{command[:destination]}",
+        duration: duration,
+        type: :upload
+      }
+    end
+
+    def execute_upload_template(command, _host_name)
+      _result, duration = measure_time do
+        with_retries(step_type: :upload_template) do
+          with_timeout do
+            @executor.upload_template(command[:source], command[:destination], command[:variables])
+          end
+        end
+      end
+      {
+        command: "upload_template: #{command[:source]} -> #{command[:destination]}",
+        duration: duration,
+        type: :upload_template
+      }
+    end
+
+    def execute_sync(command, _host_name)
+      source = command[:source]
+      destination = command[:destination]
+      fast = command.key?(:fast) ? command[:fast] : Configuration.default_sync_fast
+      parallel = command.key?(:parallel) ? command[:parallel] : Configuration.default_sync_parallel
+
+      result, duration = measure_time do
+        with_retries(step_type: :sync) do
+          with_timeout do
+            @executor.sync_directory(
+              source,
+              destination,
+              ignore: command[:ignore] || [],
+              exclude: command[:exclude] || [],
+              delete: command[:delete] || false,
+              fast: fast,
+              parallel: parallel
+            )
+          end
+        end
+      end
+
+      build_sync_result(source, destination, result, duration)
+    end
+
+    private
+
+    def build_sync_result(source, destination, result, duration)
+      {
+        command: "sync: #{source} -> #{destination}",
+        duration: duration,
+        type: :sync,
+        result: result,
+        uploaded: result[:uploaded],
+        deleted: result[:deleted],
+        total: result[:total]
+      }
+    end
+
+    def measure_time
+      start_time = Time.now
+      result = yield
+      duration = Time.now - start_time
+      [result, duration]
+    end
+
+    def with_retries(step_type: nil)
+      attempts = 0
+      max_retries = retries_for(step_type)
+      exit_codes = retry_exit_codes_for(step_type)
+      begin
+        attempts += 1
+        yield
+      rescue SSHError, SCPError, TemplateError => e
+        raise if e.is_a?(SSHError) && e.exit_status && !retry_on_exit_status?(e.exit_status, exit_codes)
+        raise if attempts > (max_retries + 1)
+
+        sleep(@retry_delay) if @retry_delay.positive?
+        retry
+      end
+    end
+
+    def with_timeout(&block)
+      return yield unless @step_timeout&.positive?
+
+      Timeout.timeout(@step_timeout, &block)
+    rescue Timeout::Error
+      raise StepTimeoutError, "exceeded #{@step_timeout}s"
+    end
+
+    def retries_for(step_type)
+      return @retries unless @retry_policy.is_a?(Hash) && step_type
+
+      policy = @retry_policy[step_type.to_s] || @retry_policy[step_type.to_sym]
+      return @retries unless policy.is_a?(Hash)
+
+      policy.fetch('retries', policy.fetch(:retries, @retries)).to_i
+    end
+
+    def retry_exit_codes_for(step_type)
+      return nil unless @retry_policy.is_a?(Hash) && step_type
+
+      policy = @retry_policy[step_type.to_s] || @retry_policy[step_type.to_sym]
+      return nil unless policy.is_a?(Hash)
+
+      policy['retry_on_exit_codes'] || policy[:retry_on_exit_codes]
+    end
+
+    def retry_on_exit_status?(exit_status, exit_codes)
+      return @retry_on_nonzero if exit_codes.nil?
+
+      Array(exit_codes).map(&:to_i).include?(exit_status.to_i)
+    end
+  end
+end

data/lib/kdeploy/{executor.rb → executor/executor.rb}

@@ -5,6 +5,8 @@ require 'net/scp'
 require 'pathname'
 require 'find'
 require 'shellwords'
+require 'tempfile'
+require 'concurrent'
 require_relative 'file_filter'
 
 module Kdeploy
@@ -16,7 +18,7 @@ module Kdeploy
       @ip = host_config[:ip]
       @password = host_config[:password]
       @key = host_config[:key]
-      @port = host_config[:port] # 新增端口支持
+      @port = host_config[:port] # Added custom port support
       @use_sudo = host_config[:use_sudo] || false
       @sudo_password = host_config[:sudo_password]
       @base_dir = host_config[:base_dir] # Base directory for resolving relative paths
@@ -114,7 +116,8 @@ module Kdeploy
       raise TemplateError.new("Template upload failed: #{e.message}", e)
     end
 
-    def sync_directory(source, destination, ignore: [], exclude: [], delete: false, use_sudo: nil)
+    def sync_directory(source, destination, ignore: [], exclude: [], delete: false, fast: nil, parallel: nil,
+                       use_sudo: nil)
       use_sudo = @use_sudo if use_sudo.nil?
 
       # Resolve relative paths relative to base_dir
@@ -127,24 +130,29 @@ module Kdeploy
       all_patterns = ignore + exclude
       filter = FileFilter.new(ignore_patterns: all_patterns)
 
+      if fast
+        rsync_result = sync_with_rsync(
+          resolved_source,
+          destination,
+          ignore: ignore,
+          exclude: exclude,
+          delete: delete,
+          use_sudo: use_sudo
+        )
+        return rsync_result if rsync_result
+      end
+
       # Collect files to sync
       files_to_sync = collect_files_to_sync(resolved_source, filter)
 
       # Upload files
-      uploaded_count = 0
-      source_path = Pathname.new(resolved_source)
-      files_to_sync.each do |file_path|
-        relative_path = Pathname.new(file_path).relative_path_from(source_path).to_s
-        remote_path = File.join(destination, relative_path).gsub(%r{/+}, '/')
-
-        # Ensure remote directory exists
-        remote_dir = File.dirname(remote_path)
-        ensure_remote_directory(remote_dir, use_sudo: use_sudo)
-
-        # Upload file
-        upload(file_path, remote_path, use_sudo: use_sudo)
-        uploaded_count += 1
-      end
+      uploaded_count = upload_files(
+        files_to_sync,
+        resolved_source,
+        destination,
+        parallel: parallel,
+        use_sudo: use_sudo
+      )
 
       # Delete extra files if requested
       deleted_count = 0
@@ -196,6 +204,80 @@ module Kdeploy
       end
     end
 
+    def sync_with_rsync(source, destination, ignore:, exclude:, delete:, use_sudo:)
+      return nil unless system('command -v rsync >/dev/null 2>&1')
+
+      exclude_file = build_rsync_excludes(ignore + exclude)
+
+      unless remote_rsync_available?
+        File.delete(exclude_file) if exclude_file && File.exist?(exclude_file)
+        return nil
+      end
+
+      begin
+        rsync_cmd = build_rsync_command(source, destination, exclude_file, delete: delete, use_sudo: use_sudo)
+        return nil unless system(rsync_cmd)
+
+        {
+          uploaded: 0,
+          deleted: 0,
+          total: 0,
+          fast_path: 'rsync'
+        }
+      ensure
+        File.delete(exclude_file) if exclude_file && File.exist?(exclude_file)
+      end
+    rescue StandardError
+      nil
+    end
+
+    def build_rsync_excludes(patterns)
+      patterns = Array(patterns).compact
+      return nil if patterns.empty?
+
+      file = Tempfile.new('kdeploy_rsync_excludes')
+      patterns.each { |pattern| file.write("#{pattern}\n") }
+      file.close
+      file.path
+    end
+
+    def build_rsync_command(source, destination, exclude_file, delete:, use_sudo:)
+      delete_flag = delete ? '--delete' : ''
+      exclude_flag = exclude_file ? "--exclude-from='#{exclude_file}'" : ''
+      ssh_cmd = build_rsync_ssh_command
+      sudo_flag = if use_sudo || requires_sudo?(destination)
+                    '--rsync-path="sudo rsync"'
+                  else
+                    ''
+                  end
+      parts = [
+        'rsync -az',
+        delete_flag,
+        exclude_flag,
+        sudo_flag,
+        "-e \"#{ssh_cmd}\"",
+        "#{Shellwords.escape(source)}/",
+        "#{@user}@#{@ip}:#{Shellwords.escape(destination)}/"
+      ].reject(&:empty?)
+      parts.join(' ')
+    end
+
+    def build_rsync_ssh_command
+      ssh = ['ssh']
+      ssh << "-p #{@port}" if @port
+      ssh << "-i #{Shellwords.escape(@key)}" if @key
+      ssh.join(' ')
+    end
+
+    def remote_rsync_available?
+      Net::SSH.start(@ip, @user, ssh_options) do |ssh|
+        output = ssh.exec!('command -v rsync 2>/dev/null || true')
+        return !output.to_s.strip.empty?
+      end
+    rescue StandardError
+      false
+    end
+
     def requires_sudo?(path)
       # Check if path is in system directories that typically require sudo
       system_dirs = %w[/etc /usr /var /opt /sbin /bin /lib /lib64 /root]
@@ -229,16 +311,16 @@ module Kdeploy
     end
 
     def wrap_with_sudo(command)
-      # 如果命令已经以 sudo 开头，不重复添加
+      # Do not add sudo again if command already starts with sudo
       return command if command.strip.start_with?('sudo')
 
-      # 多行、shell 控制结构、或包含 && / || / ; 的复合命令需整段在 sudo 下执行
+      # Wrap multi-line/control/compound commands so sudo applies to the full block
       needs_wrap = command.include?("\n") ||
                    command.match?(/\b(if|for|while|case|function)\b/) ||
                    command.match?(/\s(&&|\|\||;)\s/)
 
       if needs_wrap
-        # 转义命令中的单引号，然后用 bash -c 执行
+        # Escape single quotes in command, then execute via bash -c
         escaped_command = command.gsub("'", "'\"'\"'")
         if @sudo_password
           escaped_password = @sudo_password.gsub('\'', "'\"'\"'").gsub('$', '\\$').gsub('`', '\\`')
@@ -247,7 +329,7 @@ module Kdeploy
           "sudo bash -c '#{escaped_command}'"
         end
       elsif @sudo_password
-        # 单行命令直接包装
+        # Wrap single-line command directly
         escaped_password = @sudo_password.gsub('\'', "'\"'\"'").gsub('$', '\\$').gsub('`', '\\`')
         "echo '#{escaped_password}' | sudo -S #{command}"
       else
@@ -271,6 +353,55 @@ module Kdeploy
       files
     end
 
+    def upload_files(files_to_sync, source_dir, destination, parallel:, use_sudo:)
+      count = Concurrent::AtomicFixnum.new(0)
+      source_path = Pathname.new(source_dir)
+      parallel = normalize_parallel(parallel)
+      return upload_files_sequential(files_to_sync, source_path, destination, use_sudo, count) if parallel <= 1
+
+      queue = Queue.new
+      files_to_sync.each { |path| queue << path }
+      workers = Array.new(parallel) do
+        Thread.new do
+          until queue.empty?
+            file_path = begin
+              queue.pop(true)
+            rescue StandardError
+              nil
+            end
+            next unless file_path
+
+            upload_single_file(file_path, source_path, destination, use_sudo)
+            count.increment
+          end
+        end
+      end
+      workers.each(&:join)
+      count.value
+    end
+
+    def upload_files_sequential(files_to_sync, source_path, destination, use_sudo, count)
+      files_to_sync.each do |file_path|
+        upload_single_file(file_path, source_path, destination, use_sudo)
+        count.increment
+      end
+      count.value
+    end
+
+    def upload_single_file(file_path, source_path, destination, use_sudo)
+      relative_path = Pathname.new(file_path).relative_path_from(source_path).to_s
+      remote_path = File.join(destination, relative_path).gsub(%r{/+}, '/')
+      remote_dir = File.dirname(remote_path)
+      ensure_remote_directory(remote_dir, use_sudo: use_sudo)
+      upload(file_path, remote_path, use_sudo: use_sudo)
+    end
+
+    def normalize_parallel(parallel)
+      value = parallel.nil? ? Configuration.default_sync_parallel : parallel
+      value = value.to_i
+      value.positive? ? value : 1
+    end
+
     def ensure_remote_directory(remote_dir, use_sudo: nil)
       use_sudo = @use_sudo if use_sudo.nil?
       return if remote_dir.nil? || remote_dir.empty? || remote_dir == '.' || remote_dir == '/'

data/lib/kdeploy/{output_formatter.rb → output/output_formatter.rb}

@@ -261,6 +261,7 @@ module Kdeploy
       stats = []
       stats << @pastel.green("#{uploaded} uploaded") if uploaded.positive?
       stats << @pastel.yellow("#{deleted} deleted") if deleted.positive?
+      stats << @pastel.cyan("fast: #{result[:fast_path]}") if result[:fast_path]
       stats_str = stats.any? ? " (#{stats.join(', ')})" : " (#{total} files)"
 
       @pastel.dim('    ') + @pastel.cyan(display_path) + @pastel.dim(stats_str) + duration_str + " #{status_str}"

data/lib/kdeploy/{runner.rb → runner/runner.rb}

@@ -7,7 +7,12 @@ module Kdeploy
   class Runner
     def initialize(hosts, tasks, parallel: Configuration.default_parallel, output: ConsoleOutput.new,
                    debug: false, base_dir: nil, retries: Configuration.default_retries,
-                   retry_delay: Configuration.default_retry_delay)
+                   retry_delay: Configuration.default_retry_delay,
+                   retry_on_nonzero: Configuration.default_retry_on_nonzero,
+                   host_timeout: Configuration.default_host_timeout,
+                   step_timeout: Configuration.default_step_timeout,
+                   retry_policy: Configuration.default_retry_policy,
+                   on_step: nil)
       @hosts = hosts
       @tasks = tasks
       @parallel = parallel
@@ -16,6 +21,11 @@ module Kdeploy
       @base_dir = base_dir
       @retries = retries
       @retry_delay = retry_delay
+      @retry_on_nonzero = retry_on_nonzero
+      @host_timeout = normalize_timeout(host_timeout)
+      @step_timeout = normalize_timeout(step_timeout)
+      @retry_policy = retry_policy
+      @on_step = on_step
       @pool = Concurrent::FixedThreadPool.new(@parallel)
       @results = Concurrent::Hash.new
     end
@@ -41,59 +51,48 @@ module Kdeploy
       # If no hosts, return empty results immediately
       return @results if futures.empty?
 
-      # Collect results from futures
-      futures.each_with_index do |future, index|
-        host_name = @host_names[index] # Get host name from the stored list
-        begin
-          # Wait for future to complete and get its value
-          # This ensures the future has finished executing
-          future_result = future.value
-
-          # Handle the result
-          if future_result.nil?
-            # Future returned nil - create a default result
-            @results[host_name] = { status: :unknown, error: 'Future returned nil', output: [] }
-          elsif future_result.is_a?(Array) && future_result.length == 2
-            name, result = future_result
-            # Store the result using the name from the future
-            @results[name] = result
-          else
-            # Handle unexpected result format - create a default result
-            @results[host_name] = {
-              status: :unknown,
-              error: "Unexpected result format: #{future_result.class}",
+      pending = futures.dup
+
+      until pending.empty?
+        progressed = false
+        now = Time.now
+
+        pending.dup.each do |future|
+          meta = @future_meta[future]
+          host_name = meta[:host_name]
+          started_at = meta[:started_at].get
+
+          if future.fulfilled? || future.rejected?
+            collect_future_result(future, host_name)
+            pending.delete(future)
+            progressed = true
+          elsif timeout_exceeded?(started_at, now)
+            @results[host_name] ||= {
+              status: :failed,
+              error: "execution timeout after #{@host_timeout}s",
               output: []
             }
+            pending.delete(future)
+            progressed = true
           end
-
-          # Check if future raised an exception
-          if future.rejected?
-            error = begin
-              future.reason
-            rescue StandardError
-              'Unknown error'
-            end
-            @results[host_name] = { status: :failed, error: error, output: [] } unless @results.key?(host_name)
-          end
-        rescue StandardError => e
-          # If future.value raises an exception, create an error result
-          @results[host_name] = { status: :failed, error: "#{e.class}: #{e.message}", output: [] }
-        ensure
-          # Ensure we always have a result for this host
-          @results[host_name] ||= { status: :unknown, error: 'No result collected', output: [] }
         end
+
+        sleep(0.05) unless progressed
       end
 
       @results
     end
 
     def create_task_futures(task, task_name)
-      # Store host names in order to match with futures
-      @host_names = @hosts.keys
+      @future_meta = {}
       @hosts.map do |name, config|
-        Concurrent::Future.execute(executor: @pool) do
+        started_at = Concurrent::AtomicReference.new(nil)
+        future = Concurrent::Future.execute(executor: @pool) do
+          started_at.set(Time.now)
           execute_task_for_host(name, config, task, task_name)
         end
+        @future_meta[future] = { host_name: name, started_at: started_at }
+        future
       end
     end
 
@@ -108,7 +107,10 @@ module Kdeploy
         @output,
         debug: @debug,
         retries: @retries,
-        retry_delay: @retry_delay
+        retry_delay: @retry_delay,
+        retry_on_nonzero: @retry_on_nonzero,
+        step_timeout: @step_timeout,
+        retry_policy: @retry_policy
       )
       result = { status: :success, output: [] }
 
@@ -130,10 +132,11 @@ module Kdeploy
       commands.each do |command|
         step_result = execute_command(command_executor, command, name)
         result[:output] << step_result
+        emit_step(name, step_result, result, task_name)
       rescue StandardError => e
         step = step_description(command)
         result[:status] = :failed
-        result[:error] = "task=#{task_name} host=#{name} step=#{step} error=#{e.class}: #{e.message}"
+        result[:error] = build_step_error(task_name, name, step, e)
         result[:output] << {
           type: command[:type],
           command: step_command_string(command),
@@ -141,10 +144,58 @@ module Kdeploy
           error: "#{e.class}: #{e.message}",
           output: error_output_for_step(e)
         }
+        emit_step(name, result[:output].last, result, task_name)
         break
       end
     end
 
+    def collect_future_result(future, host_name)
+      return if @results.key?(host_name)
+
+      begin
+        future_result = future.value
+        if future_result.nil?
+          @results[host_name] = { status: :unknown, error: 'Future returned nil', output: [] }
+        elsif future_result.is_a?(Array) && future_result.length == 2
+          name, result = future_result
+          @results[name] = result
+        else
+          @results[host_name] = {
+            status: :unknown,
+            error: "Unexpected result format: #{future_result.class}",
+            output: []
+          }
+        end
+
+        if future.rejected?
+          error = begin
+            future.reason
+          rescue StandardError
+            'Unknown error'
+          end
+          @results[host_name] ||= { status: :failed, error: error, output: [] }
+        end
+      rescue StandardError => e
+        @results[host_name] = { status: :failed, error: "#{e.class}: #{e.message}", output: [] }
+      ensure
+        @results[host_name] ||= { status: :unknown, error: 'No result collected', output: [] }
+      end
+    end
+
+    def timeout_exceeded?(started_at, now)
+      return false unless @host_timeout
+      return false if started_at.nil?
+
+      (now - started_at) > @host_timeout
+    end
+
+    def normalize_timeout(timeout)
+      return nil if timeout.nil?
+
+      timeout = timeout.to_f
+      timeout.positive? ? timeout : nil
+    end
+
     def error_output_for_step(error)
       return nil unless error.is_a?(Kdeploy::SSHError)
 
@@ -156,6 +207,26 @@ module Kdeploy
       }
     end
 
+    def build_step_error(task_name, host_name, step, error)
+      base = "task=#{task_name} host=#{host_name} step=#{step} error=#{error.class}: #{error.message}"
+      return base unless error.is_a?(Kdeploy::SSHError)
+
+      if error.exit_status
+        "#{base} exit_status=#{error.exit_status} command=#{error.command}"
+      else
+        base
+      end
+    end
+
+    def emit_step(host_name, step_result, result, task_name)
+      return unless @on_step
+
+      @on_step.call(host_name, step_result, result, task_name)
+    rescue StandardError
+      # Best-effort only; never fail execution because of streaming hooks.
+      nil
+    end
+
     def execute_command(command_executor, command, host_name)
       case command[:type]
       when :run

data/lib/kdeploy/{template.rb → template/template.rb}

@@ -2,12 +2,14 @@
 
 require 'erb'
 require 'tempfile'
+require 'set'
 
 module Kdeploy
   # ERB template rendering and upload handler
   class Template
     def self.render(template_path, variables = {})
       template_content = File.read(template_path)
+      validate_template_variables(template_content, variables)
       context = create_template_context(variables)
       ERB.new(template_content).result(context.instance_eval { binding })
     end
@@ -24,17 +26,38 @@ module Kdeploy
     def self.render_and_upload(executor, template_path, destination, variables = {})
       rendered_content = render(template_path, variables)
 
-      # 创建临时文件
+      # Create temporary file
       temp_file = Tempfile.new('kdeploy')
       begin
         temp_file.write(rendered_content)
         temp_file.close
 
-        # 上传渲染后的文件
+        # Upload rendered file
         executor.upload(temp_file.path, destination)
       ensure
         temp_file.unlink
       end
     end
+
+    def self.validate_template_variables(template_content, variables)
+      required = extract_template_identifiers(template_content)
+      return if required.empty?
+
+      provided = variables.keys.to_set(&:to_s)
+      missing = required.reject { |name| provided.include?(name) }
+      return if missing.empty?
+
+      raise ArgumentError, "Missing template variables: #{missing.sort.join(', ')}"
+    end
+
+    def self.extract_template_identifiers(template_content)
+      identifiers = template_content.scan(/<%=\s*([a-zA-Z_]\w*)/).flatten
+      identifiers.uniq - ruby_keywords
+    end
+
+    def self.ruby_keywords
+      %w[alias and begin break case class def defined? do else elsif end ensure false for if in module next nil not
+         or redo rescue retry return self super then true undef unless until when while yield]
+    end
   end
 end

data/lib/kdeploy/version.rb

@@ -2,5 +2,5 @@
 
 # Kdeploy module for version management
 module Kdeploy
-  VERSION = '1.2.41' unless const_defined?(:VERSION)
+  VERSION = '1.3.3' unless const_defined?(:VERSION)
 end