aspera-cli 4.18.1 → 4.19.0

data/lib/aspera/agent/direct.rb

@@ -19,9 +19,10 @@ module Aspera
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class Direct < Base
       LISTEN_LOCAL_ADDRESS = '127.0.0.1'
-      ANY_AVAILABLE_PORT = 0 # 0 means any available port
+      # 0 means: select an available port
+      SELECT_AVAILABLE_PORT = 0
       # spellchecker: enable
-      private_constant :LISTEN_LOCAL_ADDRESS, :ANY_AVAILABLE_PORT
+      private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
 
       # method of Base
       # start ascp transfer(s) (non blocking), single or multi-session
@@ -93,12 +94,12 @@ module Aspera
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session = session.clone
             this_session[:ts] = this_session[:ts].clone
-            this_session[:env_args] = this_session[:env_args].clone
-            this_session[:env_args][:args] = this_session[:env_args][:args].clone
+            env_args = this_session[:env_args] = this_session[:env_args].clone
+            args = env_args[:args] = env_args[:args].clone
             # set multi session part
-            this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
+            args.unshift("-C#{i}:#{multi_session_info[:count]}")
             # option: increment (default as per ascp manual) or not (cluster on other side ?)
-            this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
+            args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
             # finally start the thread
             this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
             @sessions.push(this_session)
@@ -129,81 +130,46 @@ module Aspera
         Log.log.debug('fasp local shutdown')
       end
 
-      # @param event management port event
-      def process_progress(event)
-        session_id = event['SessionId']
-        case event['Type']
-        when 'INIT'
-          @pre_calc_sent = false
-          @pre_calc_last_size = nil
-          notify_progress(session_id: session_id, type: :session_start)
-        when 'NOTIFICATION' # sent from remote
-          if event.key?('PreTransferBytes')
-            @pre_calc_sent = true
-            notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
-          end
-        when 'STATS' # during transfer
-          @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
-          notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
-        when 'DONE', 'ERROR' # end of session
-          total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
-          if !@pre_calc_sent && !total_size.zero?
-            notify_progress(session_id: session_id, type: :session_size, info: total_size)
-          end
-          if @pre_calc_last_size != total_size
-            notify_progress(session_id: session_id, type: :transfer, info: total_size)
-          end
-          notify_progress(session_id: session_id, type: :end)
-          # cspell:disable
-        when 'SESSION'
-        when 'ARGSTOP'
-        when 'FILEERROR'
-        when 'STOP'
-          # cspell:enable
-          # stop event when one file is completed
-        else
-          Log.log.debug{"unknown event type #{event['Type']}"}
-        end
+      # @return [Array] list of sessions for a job
+      def sessions_by_job(job_id)
+        @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
       end
 
-      # This is the low level method to start the "ascp" process
-      # currently, relies on command line arguments
-      # start ascp with management port.
+      # This is the low level method to start the transfer process
+      # Start process with management port.
+      # returns
       # raises FaspError on error
-      # if there is a thread info: set and broadcast session id
-      # runs in separate thread
-      # @param env_args a hash containing :args :env :ascp_version
-      # @param session this session information
-      # could be private method
-      def start_transfer_with_args_env(env_args, session)
-        Aspera.assert_type(env_args, Hash)
+      # @param session this session information, keys :io and :token_regenerator
+      # @param env  [Hash]   environment variables
+      # @param name [Symbol] name of executable: :ascp, :ascp4 or :async
+      # @param args [Array]  command line arguments
+      # @return [nil] when process has exited
+      def start_and_monitor_process(
+        session:,
+        env:,
+        name:,
+        args:
+      )
         Aspera.assert_type(session, Hash)
-        Log.log.debug{"env_args=#{env_args.inspect}"}
         notify_progress(session_id: nil, type: :pre_start, info: 'starting')
         begin
-          ascp_pid = nil
+          command_pid = nil
           # we use Socket directly, instead of TCPServer, as it gives access to lower level options
-          socket_class = RUBY_ENGINE.eql?('jruby') ? ServerSocket : Socket
+          socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
           mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-          # open any available (0) local TCP port for use as ascp management port
-          mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, ANY_AVAILABLE_PORT))
+          # open any available (0) local TCP port for use as management port
+          mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
           # build arguments and add mgt port
-          ascp_arguments = ['-M', mgt_server_socket.local_address.ip_port.to_s].concat(env_args[:args])
-          # get location of ascp executable
-          ascp_path = Ascp::Installation.instance.path(env_args[:ascp_version])
-          # display ascp command line
-          Log.log.debug do
-            [
-              'execute:'.red,
-              env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
-              Shellwords.shellescape(ascp_path),
-              ascp_arguments.map{|a|Shellwords.shellescape(a)}
-            ].flatten.join(' ')
+          command_arguments = if name.eql?(:async)
+            ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
+          else
+            ['-M', mgt_server_socket.local_address.ip_port.to_s]
           end
-          # start ascp in separate process
-          ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments, close_others: true)
-          Log.log.debug{"spawned ascp pid #{ascp_pid}"}
-          notify_progress(session_id: nil, type: :pre_start, info: 'waiting for ascp')
+          command_arguments.concat(args)
+          # get location of command executable (ascp, async)
+          command_path = Ascp::Installation.instance.path(name)
+          command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
+          notify_progress(session_id: nil, type: :pre_start, info: "waiting for #{name} to start")
           mgt_server_socket.listen(1)
           # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
           Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
@@ -213,15 +179,15 @@ module Aspera
           # There is a connection to accept
           client_socket, _client_addrinfo = mgt_server_socket.accept
           Log.log.debug('after accept')
-          ascp_mgt_io = client_socket.to_io
+          management_port_io = client_socket.to_io
           # management messages include file names which may be utf8
           # by default socket is US-ASCII
           # TODO: use same value as Encoding.default_external
-          ascp_mgt_io.set_encoding(Encoding::UTF_8)
-          session[:io] = ascp_mgt_io
+          management_port_io.set_encoding(Encoding::UTF_8)
+          session[:io] = management_port_io
           processor = Ascp::Management.new
           # read management port, until socket is closed (gets returns nil)
-          while (line = ascp_mgt_io.gets)
+          while (line = management_port_io.gets)
             event = processor.process_line(line.chomp)
             next unless event
             # event is ready
@@ -241,7 +207,7 @@ module Aspera
                 # regenerate token here, expired, or error on it
                 # Note: in multi-session, each session will have a different one.
                 Log.log.warn('Regenerating token for transfer')
-                env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
+                env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
               end
               raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
             when 'DONE'
@@ -257,16 +223,16 @@ module Aspera
           raise Transfer::Error, 'transfer interrupted by user'
         ensure
           mgt_server_socket.close
-          # if ascp was successfully started, check its status
-          unless ascp_pid.nil?
+          # if command was successfully started, check its status
+          unless command_pid.nil?
             # "wait" for process to avoid zombie
-            Process.wait(ascp_pid)
+            Process.wait(command_pid)
             status = $CHILD_STATUS
-            ascp_pid = nil
+            # command_pid = nil
             session.delete(:io)
-            # status is nil if an exception occurred before starting ascp
+            # status is nil if an exception occurred before starting command
             if !status&.success?
-              message = status.nil? ? 'ascp not started' : "ascp failed (#{status})"
+              message = status.nil? ? "#{name} not started" : "#{name} failed (#{status})"
               # raise error only if there was not already an exception (ERROR_INFO)
               raise Transfer::Error, message unless $ERROR_INFO
               # else display this message also, as main exception is already here
@@ -274,11 +240,47 @@ module Aspera
             end
           end
         end
+        nil
       end
 
-      # @return [Array] list of sessions for a job
-      def sessions_by_job(job_id)
-        @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
+      private
+
+      # notify progress to callback
+      # @param event management port event
+      def process_progress(event)
+        session_id = event['SessionId']
+        case event['Type']
+        when 'INIT'
+          @pre_calc_sent = false
+          @pre_calc_last_size = nil
+          notify_progress(session_id: session_id, type: :session_start)
+        when 'NOTIFICATION' # sent from remote
+          if event.key?('PreTransferBytes')
+            @pre_calc_sent = true
+            notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
+          end
+        when 'STATS' # during transfer
+          @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
+          notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
+        when 'DONE', 'ERROR' # end of session
+          total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
+          if !@pre_calc_sent && !total_size.zero?
+            notify_progress(session_id: session_id, type: :session_size, info: total_size)
+          end
+          if @pre_calc_last_size != total_size
+            notify_progress(session_id: session_id, type: :transfer, info: total_size)
+          end
+          notify_progress(session_id: session_id, type: :end)
+          # cspell:disable
+        when 'SESSION'
+        when 'ARGSTOP'
+        when 'FILEERROR'
+        when 'STOP'
+          # cspell:enable
+          # stop event when one file is completed
+        else
+          Log.log.debug{"unknown event type #{event['Type']}"}
+        end
       end
 
       # @return [Hash] session information
@@ -289,7 +291,7 @@ module Aspera
         return matches.first
       end
 
-      # send command of management port to ascp session (used in `asession)
+      # send command to management port of command (used in `asession)
       # @param job_id identified transfer process
       # @param session_index index of session (for multi session)
       # @param data command on mgt port, examples:
@@ -309,47 +311,49 @@ module Aspera
       end
       attr_reader :sessions
 
-      private
-
       # options for initialize (same as values in option transfer_info)
-      # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
       # @param ascp_args [Array] additional arguments to ascp
-      # @param spawn_timeout_sec [Integer] timeout for ascp spawn
-      # @param spawn_delay_sec [Integer] optional delay to start between sessions
-      # @param multi_incr_udp [Boolean] true: increment udp port for each session
-      # @param trusted_certs [Array] list of files with trusted certificates (stores)
-      # @param resume [Hash] resume policy
+      # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
       # @param quiet [Boolean] by default no native ascp progress bar
+      # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
+      # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
       # @param check_ignore_cb [Proc] callback with host,port
+      # @param spawn_timeout_sec [Integer] timeout for ascp spawn
+      # @param spawn_delay_sec [Integer] optional delay to start between sessions
+      # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
+      # @param resume [Hash,NilClass] resume policy
       # @param management_cb [Proc] callback for management events
       # @param base_options [Hash] other options for base class
       def initialize(
+        ascp_args:         nil,
         wss:               true,
-        ascp_args:         [],
-        spawn_timeout_sec: 2,
-        spawn_delay_sec:   2,
-        multi_incr_udp:    true,
-        trusted_certs:     [],
-        resume:            {},
         quiet:             true,
+        trusted_certs:     nil,
+        client_ssh_key:    nil,
         check_ignore_cb:   nil,
+        spawn_timeout_sec: 2,
+        spawn_delay_sec:   2,
+        multi_incr_udp:    nil,
+        resume:            nil,
         management_cb:     nil,
         **base_options
       )
         super(**base_options)
+        # special transfer parameters
         @tr_opts = {
           ascp_args:       ascp_args,
           wss:             wss,
           quiet:           quiet,
           trusted_certs:   trusted_certs,
+          client_ssh_key:  client_ssh_key,
           check_ignore_cb: check_ignore_cb
         }
         @spawn_timeout_sec = spawn_timeout_sec
         @spawn_delay_sec = spawn_delay_sec
-        @multi_incr_udp = multi_incr_udp
-        @resume = resume
+        # default is true on windows, false on other platforms
+        @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
         @management_cb = management_cb
-        @resume_policy = Resumer.new(@resume.symbolize_keys)
+        @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
         # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
         @sessions = []
         # mutex protects global data accessed by threads
@@ -365,7 +369,7 @@ module Aspera
           Log.log.debug{"ENTER (#{Thread.current[:name]})"}
           # start transfer with selected resumer policy
           @resume_policy.execute_with_resume do
-            start_transfer_with_args_env(session[:env_args], session)
+            start_and_monitor_process(session: session, **session[:env_args])
           end
           Log.log.debug('transfer ok'.bg_green)
         rescue StandardError => e

data/lib/aspera/api/aoc.rb

@@ -439,7 +439,7 @@ module Aspera
       # create a package
       # @param package_data [Hash] package creation (with extensions...)
       # @param validate_meta [TrueClass,FalseClass] true to validate parameters locally
-      # @param new_user_option [Hash] options if an unknown user is specified
+      # @param new_user_option [Hash,NilClass] options if an unknown user is specified
       # @return transfer spec, node api and package information
       def create_package_simple(package_data, validate_meta, new_user_option)
         update_package_metadata_for_api(package_data)
@@ -476,7 +476,7 @@ module Aspera
       # callback in Node (transfer_spec_gen4)
       def add_ts_tags(transfer_spec:, app_info:)
         # translate transfer direction to upload/download
-        transfer_type = Transfer::Spec.action(transfer_spec)
+        transfer_type = Transfer::Spec.direction_to_transfer_type(transfer_spec['direction'])
         # Analytics tags
         ################
         transfer_spec.deep_merge!({

data/lib/aspera/api/httpgw.rb

@@ -45,7 +45,7 @@ module Aspera
         else
           @shared_info[:count][:sent_general] += 1
         end
-        Log.log.debug do
+        Log.log.trace1 do
           log_data = payload.dup
           log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
           "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
@@ -55,7 +55,7 @@ module Aspera
 
       # send data on http gw web socket
       def ws_send(ws_type:, data:)
-        Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
+        Log.log.trace1{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
         @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
         frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
         @ws_io.write(frame_generator.to_s)
@@ -68,13 +68,13 @@ module Aspera
                 (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
                   ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
               if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
-                Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
+                Log.log.trace1{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
               end
             end
           end
         end
         raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
-        Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
+        Log.log.trace2{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
       end
 
       # message processing for read thread
@@ -114,12 +114,12 @@ module Aspera
             # ready byte by byte until frame is ready
             # blocking read
             byte = @ws_io.read(1)
-            Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
+            Log.log.trace2{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
             frame_parser << byte
             frame_ok = frame_parser.next
             next if frame_ok.nil?
             process_received_message(frame_ok.data.to_s)
-            Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
+            Log.log.trace2{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
           rescue => e
             Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
             @shared_info[:mutex].synchronize do
@@ -139,34 +139,12 @@ module Aspera
         # identify this session uniquely
         session_id = SecureRandom.uuid
         @notify_cb&.call(session_id: nil, type: :pre_start, info: 'starting')
-        # total size of all files
-        total_bytes_to_transfer = 0
-        # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
-        files_to_read = []
-        # get source root or nil
-        source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
-        # source root is ignored by GW, used only here
-        transfer_spec.delete('source_root')
-        # compute total size of files to upload (for progress)
-        # modify transfer spec to be suitable for GW
-        transfer_spec['paths'].each do |item|
-          # save actual file location to be able read contents later
-          file_to_add = Transfer::FauxFile.open(item['source'])
-          if file_to_add
-            item['source'] = file_to_add.path
-            item['file_size'] = file_to_add.size
-          else
-            file_to_add = item['source']
-            # add source root if needed
-            file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
-            # GW expects a simple file name in 'source' but if user wants to change the name, we take it
-            item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
-            item['file_size'] = File.size(file_to_add)
-          end
-          # save so that we can actually read the file later
-          files_to_read.push(file_to_add)
-          total_bytes_to_transfer += item['file_size']
-        end
+        # process files to send, modify `paths` in transfer_spec
+        files_to_send = process_upload_list(transfer_spec)
+        # total size of all files is last element
+        total_bytes_to_transfer = files_to_send.pop
+        Log.log.trace1{Log.dump(:modified_tspec, transfer_spec)}
+        Log.log.trace1{Log.dump(:files_to_send, files_to_send)}
         # TODO: check that this is available in endpoints: @api_info['endpoints']
         upload_url = File.join(@gw_root_url, @upload_version, 'upload')
         @notify_cb&.call(session_id: nil, type: :pre_start, info: 'connecting wss')
@@ -180,11 +158,6 @@ module Aspera
         @ws_handshake << @ws_io.readuntil("\r\n\r\n")
         Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
         Log.log.debug{"#{LOG_WS_SEND}handshake success"}
-        # start read thread after handshake
-        @ws_read_thread = Thread.new {process_read_thread}
-        @notify_cb&.call(session_id: session_id, type: :session_start)
-        @notify_cb&.call(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
-        sleep(1)
         # data shared between main thread and read thread
         @shared_info = {
           read_exception: nil, # error message if any in callback
@@ -197,34 +170,34 @@ module Aspera
           mutex:          Mutex.new,
           cond_var:       ConditionVariable.new
         }
+        # start read thread after handshake
+        @ws_read_thread = Thread.new {process_read_thread}
+        @notify_cb&.call(session_id: session_id, type: :session_start)
+        @notify_cb&.call(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
+        sleep(1)
         # notify progress bar
         @notify_cb&.call(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
         # first step send transfer spec
-        Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
         ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
         # current file index
         file_index = 0
         # aggregate size sent
         session_sent_bytes = 0
         # process each file
-        transfer_spec['paths'].each do |item|
+        files_to_send.each do |file_to_send|
+          last_slice = (file_to_send[:size] - 1) / @upload_chunk_size
           slice_info = {
-            name:       nil,
+            name:         file_to_send[:name],
             # TODO: get mime type?
-            type:       'application/octet-stream',
-            size:       item['file_size'],
-            slice:      0, # current slice index
+            type:         'application/octet-stream',
+            size:         file_to_send[:size],
+            slice:        0, # current slice index
             # index of last slice (i.e number of slices - 1)
-            last_slice: (item['file_size'] - 1) / @upload_chunk_size,
-            fileIndex:  file_index
+            total_slices: last_slice + 1,
+            fileIndex:    file_index
           }
-          file = files_to_read[file_index]
-          if file.is_a?(Transfer::FauxFile)
-            slice_info[:name] = file.path
-          else
-            file = File.open(file)
-            slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
-          end
+          file = file_to_send[:file]
+          file = File.open(file) unless file.is_a?(Transfer::FauxFile)
           begin
             until file.eof?
               slice_bin_data = file.read(@upload_chunk_size)
@@ -238,9 +211,9 @@ module Aspera
                   # send once, before data, at beginning
                   ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
                   ws_send(ws_type: :binary, data: slice_bin_data)
-                  Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
+                  Log.log.trace1{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{last_slice}"}
                   # send once, after data, at end
-                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(last_slice)
                 end
               rescue Errno::EPIPE => e
                 raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
@@ -337,6 +310,68 @@ module Aspera
           end
         end
       end
+
+      private
+
+      # compute total size of files to upload (for progress)
+      # modify transfer spec to be suitable for HTTPGW
+      # @param transfer_spec [Hash] transfer specification
+      # @return [Array] info on files to send
+      def process_upload_list(transfer_spec)
+        total_bytes_to_transfer = 0
+        source_prefix = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] + '/' : ''
+        files_to_send = []
+        transfer_spec['paths'].each do |one_path|
+          source_path = source_prefix + one_path['source']
+          faux_file = Transfer::FauxFile.create(source_path)
+          if faux_file
+            total_bytes_to_transfer += faux_file.size
+            files_to_send.push({
+              file: faux_file,
+              name: faux_file.path,
+              size: faux_file.size
+            })
+          elsif File.file?(source_path)
+            # regular file
+            file_size = File.size(source_path)
+            total_bytes_to_transfer += file_size
+            files_to_send.push({
+              file: source_path,
+              # GW expects a simple file name in 'source' but if user wants to change the name, we take it
+              name: File.basename(one_path['destination'].nil? ? source_path : one_path['destination']),
+              size: file_size
+            })
+          elsif File.directory?(source_path)
+            folders_to_process = [source_path]
+            until folders_to_process.empty?
+              folder = folders_to_process.shift
+              # read all entries
+              Dir.entries(folder).each do |entry|
+                next if entry.eql?('.') || entry.eql?('..')
+                entry_path = File.join(folder, entry)
+                if File.directory?(entry_path)
+                  folders_to_process.push(entry_path)
+                elsif File.file?(entry_path)
+                  file_size = File.size(entry_path)
+                  total_bytes_to_transfer += file_size
+                  files_to_send.push({
+                    file: entry_path,
+                    name: entry_path,
+                    size: file_size
+                  })
+                else
+                  Log.log.warn{"Ignoring non file/directory: #{entry_path}"}
+                end
+              end
+            end
+          else
+            raise "File not found: #{source_path}"
+          end
+        end
+        transfer_spec['paths'] = files_to_send.map{|i|{'source' => i[:name]}}
+        files_to_send.push(total_bytes_to_transfer)
+        return files_to_send
+      end
     end
   end
 end