aspera-cli 4.24.1 → 4.24.2

data/lib/aspera/sync/operations.rb

@@ -18,16 +18,16 @@ module Aspera
   module Sync
     # builds command line arg for async and execute it
     module Operations
-      # sync direction
+      # Sync direction
       DIRECTIONS = %i[push pull bidi].freeze
-      # default direction for sync
-      DEFAULT_DIRECTION = :push
+      # Default direction for sync
+      DEFAULT_DIRECTION = DIRECTIONS.first
 
       class << self
         # Set `remote_dir` in sync parameters based on transfer spec
-        # @param params [Hash] Sync parameters, old or new format
-        # @param remote_dir_key [String] key to update in above hash
-        # @param transfer_spec [Hash] transfer spec
+        # @param params         [Hash]   Sync parameters, in `conf` or `args` format.
+        # @param remote_dir_key [String] Key to update in above hash
+        # @param transfer_spec  [Hash]   Transfer spec
         def update_remote_dir(params, remote_dir_key, transfer_spec)
           if transfer_spec.dig(*%w[tags aspera node file_id])
             # in AoC, use gen4
@@ -126,21 +126,21 @@ module Aspera
                 transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
                 Log.dump(:auth_ts, transfer_spec)
                 transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
-                tspec_to_sync_info(transfer_spec, session, SESSION_SCHEMA)
+                tspec_to_sync_info(transfer_spec, session, ARGS_SESSION_SCHEMA)
                 session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
                 update_remote_dir(session, 'remote_dir', transfer_spec)
               end
             end
             if params.key?('instance')
               Aspera.assert_type(params['instance'], Hash)
-              instance_builder = CommandLineBuilder.new(params['instance'], INSTANCE_SCHEMA, CommandLineConverter)
+              instance_builder = CommandLineBuilder.new(params['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
               instance_builder.process_params
               instance_builder.add_env_args(env_args)
             end
             params['sessions'].each do |session_params|
               Aspera.assert_type(session_params, Hash)
               Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
-              session_builder = CommandLineBuilder.new(session_params, SESSION_SCHEMA, CommandLineConverter)
+              session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
               session_builder.process_params
               session_builder.add_env_args(env_args)
             end
@@ -277,20 +277,20 @@ module Aspera
       end
       # Private stuff:
       # Read JSON schema and mapping to command line options
-      INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'args')
-      SESSION_SCHEMA = INSTANCE_SCHEMA['properties']['sessions']['items']
-      INSTANCE_SCHEMA['properties'].delete('sessions')
+      ARGS_INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'args')
+      ARGS_SESSION_SCHEMA = ARGS_INSTANCE_SCHEMA['properties']['sessions']['items']
+      ARGS_INSTANCE_SCHEMA['properties'].delete('sessions')
       CONF_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'conf')
-      CommandLineBuilder.adjust_properties_defaults(INSTANCE_SCHEMA['properties'])
-      CommandLineBuilder.adjust_properties_defaults(SESSION_SCHEMA['properties'])
-      CommandLineBuilder.adjust_properties_defaults(CONF_SCHEMA['properties'])
+      CommandLineBuilder.validate_schema(ARGS_INSTANCE_SCHEMA)
+      CommandLineBuilder.validate_schema(ARGS_SESSION_SCHEMA)
+      CommandLineBuilder.validate_schema(CONF_SCHEMA)
       CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
       ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
       PRIVATE_FOLDER = '.private-asp'
       ASYNC_DB = 'snap.db'
       PARAM_KEYS = %w[local sessions].freeze
 
-      private_constant :INSTANCE_SCHEMA, :SESSION_SCHEMA, :CONF_SCHEMA, :CMDLINE_PARAMS_KEYS, :ASYNC_ADMIN_EXECUTABLE, :PRIVATE_FOLDER, :ASYNC_DB, :PARAM_KEYS
+      private_constant :ARGS_INSTANCE_SCHEMA, :ARGS_SESSION_SCHEMA, :CMDLINE_PARAMS_KEYS, :ASYNC_ADMIN_EXECUTABLE, :PRIVATE_FOLDER, :ASYNC_DB, :PARAM_KEYS
     end
   end
 end

data/lib/aspera/temp_file_manager.rb

@@ -17,17 +17,29 @@ module Aspera
     private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
 
     attr_accessor :cleanup_on_exit
+    attr_reader :global_temp
 
     def initialize
       @created_files = []
       @cleanup_on_exit = true
+      @global_temp = Etc.systmpdir
+    end
+
+    def global_temp=(value)
+      @global_temp = case value
+      when '@env' then Dir.tmpdir
+      when '@sys' then Etc.systmpdir
+      else value
+      end
     end
 
     def delete_file(filepath)
       File.delete(filepath) if @cleanup_on_exit
+    rescue => e
+      Log.log.warn{"Problem deleting file: #{filepath}: #{e.message}"}
     end
 
-    # call this on process exit
+    # Call this on process exit
     def cleanup
       @created_files.each do |filepath|
         delete_file(filepath) if File.file?(filepath)
@@ -35,7 +47,7 @@ module Aspera
       @created_files = []
     end
 
-    # ensure that provided folder exists, or create it, generate a unique filename
+    # Ensure that provided folder exists, or create it, generate a unique filename
     # @return path to that unique file
     def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
       FileUtils.mkdir_p(temp_folder)
@@ -44,7 +56,7 @@ module Aspera
       new_file
     end
 
-    # same as above but in global temp folder, with user's name
+    # Same as above but in global temp folder, with user's name
     def new_file_path_global(prefix = nil, suffix: nil)
       username =
         begin
@@ -53,11 +65,11 @@ module Aspera
           'unknown_user'
         end
       prefix = [prefix, username].compact.join('-')
-      new_file_path_in_folder(Etc.systmpdir, prefix: prefix, suffix: suffix)
+      new_file_path_in_folder(@global_temp, prefix: prefix, suffix: suffix)
     end
 
+    # Garbage collect undeleted files
     def cleanup_expired(temp_folder)
-      # garbage collect undeleted files
       Dir.entries(temp_folder).each do |name|
         file_path = File.join(temp_folder, name)
         age_sec = (Time.now - File.stat(file_path).mtime).to_i

data/lib/aspera/transfer/error.rb

@@ -1,24 +1,33 @@
 # frozen_string_literal: true
 
-require 'aspera/transfer/error_info'
+require 'aspera/ascp/management'
 
 module Aspera
   module Transfer
-    # error raised if transfer fails
+    # Error raised if transfer fails
     class Error < StandardError
+      # Error code like on management port
       attr_reader :err_code
 
-      def initialize(message, err_code = nil)
-        super(message)
-        @err_code = err_code
+      # @param description [String] `Description` on management port
+      # @param code [Integer] `Description` on management port, use zero if unknown
+      def initialize(description, code: nil)
+        super(description)
+        @err_code = code.to_i
       end
 
+      # @return [Hash] Information on that error
       def info
-        r = ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
+        r = Ascp::Management::ERRORS[@err_code] || Ascp::Management::ERRORS[0]
         return r.merge({i: @err_code})
       end
 
-      def retryable?; info[:r]; end
+      # Is that transfer error retryable ?
+      # @param message [String, nil] Optional actual message on management port
+      def retryable?
+        return false if @err_code.eql?(14) && message.eql?('Target address not available')
+        info[:r]
+      end
     end
   end
 end

data/lib/aspera/transfer/parameters.rb

@@ -18,11 +18,8 @@ require 'openssl'
 
 module Aspera
   module Transfer
-    # translate transfer specification to ascp parameter list
+    # Translate transfer specification to `ascp` parameter list
     class Parameters
-      # `ascp` options to provide a file list
-      FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
-      private_constant :FILE_LIST_OPTIONS
       HTTP_FALLBACK_ACTIVATION_VALUES = ['1', 1, true, 'force'].freeze
 
       class << self
@@ -30,7 +27,6 @@ module Aspera
         def file_list_folder=(value)
           @file_list_folder = value
           return if @file_list_folder.nil?
-
           FileUtils.mkdir_p(@file_list_folder)
           TempFileManager.instance.cleanup_expired(@file_list_folder)
         end
@@ -42,14 +38,19 @@ module Aspera
           @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
         end
 
-        # file list is provided directly with ascp arguments
+        # File list is provided directly with ascp arguments
         # @columns ascp_args [Array,NilClass] ascp arguments
         def ascp_args_file_list?(ascp_args)
           ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
         end
       end
 
-      # @columns options [Hash] key: :wss: bool, :ascp_args: array of strings
+      # @param job_spec        [Hash]   Transfer spec
+      # @param ascp_args       [Array]  Other ascp args
+      # @param quiet           [Bool]   Remove ascp output
+      # @param trusted_certs   [Array]  Trusted certificates
+      # @param client_ssh_key  [Symbol] :rsa or :dsa
+      # @param check_ignore_cb [Proc]   Callback
       def initialize(
         job_spec,
         ascp_args:       nil,
@@ -60,17 +61,17 @@ module Aspera
         check_ignore_cb: nil
       )
         @job_spec = job_spec
+        Aspera.assert_type(@job_spec, Hash)
         @ascp_args = ascp_args.nil? ? [] : ascp_args
+        Aspera.assert_type(@ascp_args, Array){'ascp_args'}
+        Aspera.assert(@ascp_args.all?(String)){'all ascp arguments must be String'}
         @wss = wss
         @quiet = quiet
         @trusted_certs = trusted_certs.nil? ? [] : trusted_certs
-        @client_ssh_key = client_ssh_key.nil? ? :rsa : client_ssh_key.to_sym
-        @check_ignore_cb = check_ignore_cb
-        Aspera.assert_type(@job_spec, Hash)
-        Aspera.assert_type(@ascp_args, Array){'ascp_args'}
-        Aspera.assert(@ascp_args.all?(String)){'all ascp arguments must be String'}
         Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
+        @client_ssh_key = client_ssh_key.nil? ? :rsa : client_ssh_key.to_sym
         Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
+        @check_ignore_cb = check_ignore_cb
         @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
       end
 
@@ -109,7 +110,7 @@ module Aspera
             Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
           end
         end
-        @builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
+        @builder.add_command_line_options("#{file_list_option}=#{file_list_file}") unless file_list_option.nil?
       end
 
       # @return the list of certificates (option `-i`) to use when token/ssh or wss are used
@@ -118,7 +119,7 @@ module Aspera
         # use web socket secure for session ?
         if @builder.read_param('wss_enabled') && (@wss || !@job_spec.key?('fasp_port'))
           # by default use web socket session if available, unless removed by user
-          @builder.add_command_line_options(['--ws-connect'])
+          @builder.add_command_line_options('--ws-connect')
           # TODO: option to give order ssh,ws (legacy http is implied by ssh)
           # This will need to be cleaned up in aspera core
           @job_spec['ssh_port'] = @builder.read_param('wss_port')
@@ -148,7 +149,7 @@ module Aspera
         return certificates_to_use
       end
 
-      # translate transfer spec to env vars and command line arguments for ascp
+      # Translate transfer spec to env vars and command line arguments for `ascp`
       def ascp_args
         env_args = {
           args: [],
@@ -156,18 +157,27 @@ module Aspera
           name: :ascp
         }
 
-        # special cases
+        # Special cases
         @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
 
-        # notify multi-session was already used, anyway it was deleted by agent direct
+        # Notify multi-session was already used, anyway it was deleted by agent direct
         Aspera.assert(!@builder.read_param('multi_session'))
 
-        # add ssh or wss certificates
+        # Add ssh or wss certificates
         # (reverse, to keep order, as we unshift)
         remote_certificates&.reverse_each do |cert|
           env_args[:args].unshift('-i', cert)
         end
 
+        case (delete_source = @builder.read_param('delete_source'))
+        when true
+          DELETE_EQUIV.each{ |i| @job_spec[i] = true}
+        when false
+          DELETE_EQUIV.each{ |i| @job_spec.delete(i)}
+        when nil
+        else Aspera.error_unexpected_value(delete_source){'delete_source'}
+        end
+
         # process parameters as specified in table
         @builder.process_params
 
@@ -180,7 +190,7 @@ module Aspera
           base64_destination = true
         end
         # destination will be base64 encoded, put this before source path arguments
-        @builder.add_command_line_options(['--dest64']) if base64_destination
+        @builder.add_command_line_options('--dest64') if base64_destination
         # optional arguments, at the end to override previous ones (to allow override)
         @builder.add_command_line_options(@ascp_args)
         # get list of source files to transfer and build arg for ascp
@@ -190,7 +200,7 @@ module Aspera
         # ascp4 does not support base64 encoding of destination
         destination_folder = Base64.strict_encode64(destination_folder) if base64_destination
         # destination MUST be last command line argument to ascp
-        @builder.add_command_line_options([destination_folder])
+        @builder.add_command_line_options(destination_folder)
         @builder.add_env_args(env_args)
         env_args[:args].unshift('-q') if @quiet
         # add fallback cert and key as arguments if needed
@@ -203,6 +213,10 @@ module Aspera
         Log.log.debug{"ascp args: #{env_args}"}
         return env_args
       end
+      DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
+      # `ascp` options to provide a file list
+      FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
+      private_constant :DELETE_EQUIV, :FILE_LIST_OPTIONS
     end
   end
 end

data/lib/aspera/transfer/resumer.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'aspera/log'
+require 'aspera/assert'
+require 'aspera/transfer/error'
+
+module Aspera
+  module Transfer
+    # Implements a simple resume policy
+    class Resumer
+      # @param iter_max      [Integer] Maximum number of executions
+      # @param sleep_initial [Integer] Initial wait to re-execute
+      # @param sleep_factor  [Integer] Multiplier
+      # @param sleep_max.    [Integer] Max iterations
+      def initialize(
+        iter_max: 7,
+        sleep_initial: 2,
+        sleep_factor:  2,
+        sleep_max:     60
+      )
+        Aspera.assert_type(iter_max, Integer){k}
+        @iter_max = iter_max
+        Aspera.assert_type(sleep_initial, Integer){k}
+        @sleep_initial = sleep_initial
+        Aspera.assert_type(sleep_factor, Integer){k}
+        @sleep_factor = sleep_factor
+        Aspera.assert_type(sleep_max, Integer){k}
+        @sleep_max = sleep_max
+      end
+
+      # Calls block a number of times (resumes) until success or limit reached
+      # This is re-entrant, one resumer can handle multiple transfers in //
+      #
+      # @param block [Proc]
+      def execute_with_resume
+        Aspera.assert(block_given?)
+        # maximum of retry
+        remaining_resumes = @iter_max
+        sleep_seconds = @sleep_initial
+        Log.log.debug{"retries=#{remaining_resumes}"}
+        # try to send the file until ascp is successful
+        loop do
+          Log.log.debug('Transfer session starting')
+          begin
+            # Call provided block: execute transfer
+            yield
+            # Exit retry loop if success
+            break
+          rescue Error => e
+            Log.log.warn{"A transfer error occurred during transfer: #{e.message}"}
+            Log.log.debug{"Retryable ? #{e.retryable?}"}
+            # do not retry non-retryable
+            raise unless e.retryable?
+            # exit if we exceed the max number of retry
+            raise Error, "Maximum number of retry reached: #{@iter_max}" if remaining_resumes <= 0
+          end
+
+          # take this retry in account
+          remaining_resumes -= 1
+          Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
+
+          # wait a bit before retrying, maybe network condition will be better
+          sleep(sleep_seconds)
+
+          # increase retry period
+          sleep_seconds *= @sleep_factor
+          # cap value
+          sleep_seconds = @sleep_max if sleep_seconds > @sleep_max
+        end
+      end
+    end
+  end
+end

data/lib/aspera/transfer/spec.rb

@@ -5,7 +5,8 @@ require 'aspera/assert'
 
 module Aspera
   module Transfer
-    # parameters for Transfer Spec
+    # Parameters for Transfer Spec
+    # Parameters are generated from JSON Schema.
     class Spec
       # default transfer username for access key based transfers
       ACCESS_KEY_TRANSFER_USER = 'xfer'
@@ -44,12 +45,12 @@ module Aspera
         end
 
         def fix_transferd_resume_policy(transfer_spec)
-          # Fix discrepency in transfer spec
+          # Fix discrepancy in transfer spec
           transfer_spec['resume_policy'] = POLICY_FIX[transfer_spec['resume_policy']] if transfer_spec.key?('resume_policy')
         end
       end
       SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'spec')
-      CommandLineBuilder.adjust_properties_defaults(SCHEMA['properties'])
+      CommandLineBuilder.validate_schema(SCHEMA, ascp: true)
       # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
       SCHEMA['properties'].each do |name, description|
         next unless description['enum'].is_a?(Array)