aspera-cli 4.7.0 → 4.9.0

data/lib/aspera/fasp/parameters.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/log'
 require 'aspera/command_line_builder'
 require 'aspera/temp_file_manager'
@@ -14,40 +15,45 @@ module Aspera
     # translate transfer specification to ascp parameter list
     class Parameters
       # Agents shown in manual for parameters (sub list)
-      SUPPORTED_AGENTS=[:direct,:node,:connect]
+      SUPPORTED_AGENTS = %i[direct node connect].freeze
       # Short names of columns in manual
-      SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
+      SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
+
+      private_constant :SUPPORTED_AGENTS
 
       class << self
         # Temp folder for file lists, must contain only file lists
         # because of garbage collection takes any file there
         # this could be refined, as , for instance, on macos, temp folder is already user specific
-        @file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-        @param_description_cache=nil
+        @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists')
+        @param_description_cache = nil
         # @return normalized description of transfer spec parameters, direct from yaml
         def description
-          return @param_description_cache unless @param_description_cache.nil?
-          # config file in same folder with same name as this source
-          @param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
-          Aspera::CommandLineBuilder.normalize_description(@param_description_cache)
+          if @param_description_cache.nil?
+            # config file in same folder with same name as this source
+            description_from_yaml=YAML.load_file("#{__FILE__[0..-3]}yaml")
+            @param_description_cache = Aspera::CommandLineBuilder.normalize_description(description_from_yaml)
+          end
+          return @param_description_cache
         end
 
         # @return a table suitable to display in manual
         def man_table
-          result=[]
+          result = []
           description.each do |k,i|
-            param={name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
+            param = {name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
+            # add flags for supported agents in doc
             SUPPORTED_AGENTS.each do |a|
-              param[a.to_s[0].to_sym]=i[:tragents].nil? || i[:tragents].include?(a) ? 'Y' : ''
+              param[a.to_s[0].to_sym] = i[:tragents].nil? || i[:tragents].include?(a) ? 'Y' : ''
             end
             # only keep lines that are usable in supported agents
             next if SUPPORTED_AGENTS_SHORT.inject(true){|m,j|m && param[j].empty?}
-            param[:cli]=
-            case i[:cltype]
-            when :envvar then 'env:'+i[:clvarname]
-            when :opt_without_arg,:opt_with_arg then i[:clswitch]
-            else ''
-            end
+            param[:cli] =
+              case i[:cltype]
+              when :envvar then 'env:' + i[:clvarname]
+              when :opt_without_arg,:opt_with_arg then i[:clswitch]
+              else ''
+              end
             if i.has_key?(:enum)
               param[:description] += "\nAllowed values: #{i[:enum].join(', ')}"
             end
@@ -65,17 +71,20 @@ module Aspera
         # special encoding methods used in YAML (key: :clconvert)
         def clconv_base64(v); Base64.strict_encode64(v); end
 
-        def ts_has_file_list(ts)
-          ts.has_key?('EX_ascp_args') and ts['EX_ascp_args'].is_a?(Array) and ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}
+        # file list is provided directly with ascp arguments
+        def ts_has_ascp_file_list(ts)
+          (ts['EX_ascp_args'].is_a?(Array) && ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}) ||
+          ts.has_key?('EX_file_list') ||
+          ts.has_key?('EX_file_pair_list')
         end
 
         def ts_to_env_args(transfer_spec,options)
-          return Parameters.new(transfer_spec,options).ascp_args()
+          return Parameters.new(transfer_spec,options).ascp_args
         end
 
         # temp file list files are created here
         def file_list_folder=(v)
-          @file_list_folder=v
+          @file_list_folder = v
           return if @file_list_folder.nil?
           FileUtils.mkdir_p(@file_list_folder)
           TempFileManager.instance.cleanup_expired(@file_list_folder)
@@ -87,18 +96,18 @@ module Aspera
 
       # @param options [Hash] key: :wss: bool
       def initialize(job_spec,options)
-        @job_spec=job_spec
-        @options=options
-        @builder=Aspera::CommandLineBuilder.new(@job_spec,self.class.description)
+        @job_spec = job_spec
+        @options = options
+        @builder = Aspera::CommandLineBuilder.new(@job_spec,self.class.description)
         Log.log.debug("agent options: #{@options}")
       end
 
       # translate transfer spec to env vars and command line arguments for ascp
       # NOTE: parameters starting with "EX_" (extended) are not standard
       def ascp_args
-        env_args={
-          args: [],
-          env: {},
+        env_args = {
+          args:         [],
+          env:          {},
           ascp_version: :ascp
         }
         # some ssh credentials are required to avoid interactive password input
@@ -117,12 +126,12 @@ module Aspera
           @builder.add_command_line_options(['--ws-connect'])
           # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
           # quel bordel:
-          @job_spec['ssh_port']=@builder.process_param('wss_port',:get_value)
+          @job_spec['ssh_port'] = @builder.process_param('wss_port',:get_value)
           @job_spec.delete('fasp_port')
           @job_spec.delete('EX_ssh_key_paths')
           @job_spec.delete('sshfp')
           # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
-          @job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
+          @job_spec['EX_ssh_key_paths'] = [OpenSSL::X509::DEFAULT_CERT_FILE]
           Log.log.debug('CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl')
         else
           # remove unused parameter (avoid warning)
@@ -140,48 +149,50 @@ module Aspera
           # destination will be base64 encoded, put before path arguments
           @builder.add_command_line_options(['--dest64'])
         end
-        # paths is mandatory, unless ...
-        file_list_provided=self.class.ts_has_file_list(@job_spec)
-        @builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive') and !file_list_provided
-        paths_array=@builder.process_param('paths',:get_value)
-        if file_list_provided && !paths_array.nil?
-          Log.log.warn('file list provided both in transfer spec and ascp file list. Keeping file list only.')
-          paths_array=nil
-        end
-        if !paths_array.nil?
-          # it's an array
-          raise 'paths is empty in transfer spec' if paths_array.empty?
-          # use file list if there is storage defined for it.
-          if self.class.file_list_folder.nil?
-            # not safe for special characters ? (maybe not, depends on OS)
-            Log.log.debug('placing source file list on command line (no file list file)')
-            @builder.add_command_line_options(paths_array.map{|i|i['source']})
+        # process file lists
+        begin
+          # is the file list provided through EX_ parameters?
+          ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec)
+          # set if paths is mandatory in ts
+          @builder.params_definition['paths'][:mandatory] = !@job_spec.has_key?('keepalive') && !ascp_file_list_provided
+          # get paths in transfer spec (after setting if it is mandatory)
+          ts_paths_array = @builder.process_param('paths',:get_value)
+          if ascp_file_list_provided && !ts_paths_array.nil?
+            raise 'file list provided both in transfer spec and ascp file list. Remove one of them.'
+          end
+          # option 1: EX_file_list
+          file_list_file = @builder.process_param('EX_file_list',:get_value)
+          if !file_list_file.nil?
+            option = '--file-list'
           else
-            file_list_file=@builder.process_param('EX_file_list',:get_value)
+            # option 2: EX_file_pair_list
+            file_list_file = @builder.process_param('EX_file_pair_list',:get_value)
             if !file_list_file.nil?
-              option='--file-list'
-            else
-              file_list_file=@builder.process_param('EX_file_pair_list',:get_value)
-              if !file_list_file.nil?
-                option='--file-pair-list'
-              else
-                # safer option: file list
+              option = '--file-pair-list'
+            elsif !ts_paths_array.nil?
+              # option 3: in TS, it is an array
+              if !self.class.file_list_folder.nil?
+                # safer option: generate a file list file if there is storage defined for it
                 # if there is destination in paths, then use filepairlist
                 # TODO: well, we test only the first one, but anyway it shall be consistent
-                if paths_array.first.has_key?('destination')
-                  option='--file-pair-list'
-                  lines=paths_array.each_with_object([]){|e,m|m.push(e['source'],e['destination']);}
+                if ts_paths_array.first.has_key?('destination')
+                  option = '--file-pair-list'
+                  lines = ts_paths_array.each_with_object([]){|e,m|m.push(e['source'],e['destination']);}
                 else
-                  option='--file-list'
-                  lines=paths_array.map{|i|i['source']}
+                  option = '--file-list'
+                  lines = ts_paths_array.map{|i|i['source']}
                 end
-                file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
+                file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
                 File.write(file_list_file, lines.join("\n"))
                 Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
+              else
+                # not safe for special characters ? (maybe not, depends on OS)
+                Log.log.debug('placing source file list on command line (no file list file)')
+                @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
               end
             end
-            @builder.add_command_line_options(["#{option}=#{file_list_file}"])
           end
+          @builder.add_command_line_options(["#{option}=#{file_list_file}"]) unless option.nil?
         end
         # optional args, at the end to override previous ones (to allow override)
         @builder.add_command_line_options(@builder.process_param('EX_ascp_args',:get_value))

data/lib/aspera/fasp/parameters.yaml

@@ -49,7 +49,13 @@ create_dir:
   :cltype: :opt_without_arg
   :clswitch: "-d"
 delete_before_transfer:
-  :desc: "Before transfer, delete files that exist at the destination but not at the source. The source and destination arguments must be directories that have matching names. Objects on the destination that have the same name but different type or size as objects on the source are not deleted."
+  :desc: |-
+    Before transfer, delete files that exist at the destination but not at the source.
+    The source and destination arguments must be directories that have matching names.
+    Objects on the destination that have the same name but different type or size as objects
+    on the source are not deleted.
+
+
   :cltype: :opt_without_arg
 delete_source: # duplicate of remove_after_transfer ?
   :desc: Remove SRC files after transfer success
@@ -138,16 +144,24 @@ https_fallback_port:
   :cltype: :opt_with_arg
   :clswitch: "-t"
 move_after_transfer:
+  :desc: The relative path to which the files will be moved after the transfer at the source side.
   :cltype: :opt_with_arg
 multi_session:
-  :desc: "Use multi-session transfer. max 128.\n
-Each participant on one host needs an independent UDP (-O) port.\n
-Large files are split between sessions only when transferring with resume_policy=none."
+  :desc: |
+    Use multi-session transfer. max 128.
+    Each participant on one host needs an independent UDP (-O) port.
+    Large files are split between sessions only when transferring with resume_policy=none.
+
+
   :accepted_types: :int
   :cltype: :ignore
   :clswitch: "-C"
 multi_session_threshold:
-  :desc: Split files across multiple ascp sessions if their size in bytes is greater than or equal to the specified value. (0=no file is split)
+  :desc: |-
+    Split files across multiple ascp sessions if their size in bytes is greater than or equal to the specified value.
+    (0=no file is split)
+
+
   :accepted_types: :int
   :tragents:
   - :direct
@@ -163,6 +177,18 @@ overwrite:
   - older
   - diff+older
   :cltype: :opt_with_arg
+password:
+  :desc: |-
+    Password for local Windows user when transfer user associated with node api user is not the same as the one running asperanoded.
+    Allows impersonating the transfer user and have access to resources (e.g. network shares).
+    Windows only, node api only.
+
+
+  :required: false
+  :accepted_types: :string
+  :cltype: :ignore
+  :tragents:
+  - :node
 paths:
   :desc: Array of path to the source (required) and a path to the destination (optional).
   :required: true
@@ -233,10 +259,13 @@ remove_skipped:
   - :node
   :cltype: :opt_without_arg
 proxy:
-  :desc: "Specify the address of the Aspera high-speed proxy server.\n
-dnat(s)://[user[:password]@]server:port\n
-Default ports for DNAT and DNATS protocols are 9091 and 9092.\n
-Password, if specified here, overrides the value of environment variable ASPERA_PROXY_PASS."
+  :desc: |-
+    Specify the address of the Aspera high-speed proxy server.
+    dnat(s)://[user[:password]@]server:port
+    Default ports for DNAT and DNATS protocols are 9091 and 9092.
+    Password, if specified here, overrides the value of environment variable ASPERA_PROXY_PASS.
+
+
   :tragents:
   - :direct
   - :sdk
@@ -279,9 +308,12 @@ ssh_port:
   :cltype: :opt_with_arg
   :clswitch: "-P"
 ssh_private_key:
-  :desc: "Private key used for SSH authentication.\n
-   Shall look like: -----BEGIN RSA PRIV4TE KEY-----\\nMII...\n
-   Note the JSON encoding: \\n for newlines."
+  :desc: |-
+    Private key used for SSH authentication.
+    Shall look like: -----BEGIN RSA PRIV4TE KEY-----\nMII...
+    Note the JSON encoding: \n for newlines.
+
+
   :tragents:
   - :direct
   - :sdk
@@ -355,8 +387,11 @@ use_system_ssh:
   :cltype: :ignore
   :clswitch: "-SSH"
 source_root:
-  :desc: "Path to be prepended to each source path.\n
-This is either a conventional path or it can be a URI but only if there is no root defined."
+  :desc: |-
+    Path to be prepended to each source path.
+    This is either a conventional path or it can be a URI but only if there is no root defined.
+
+
   :cltype: :opt_with_arg
   :clswitch: "--source-prefix64"
   :clconvert: Aspera::Fasp::Parameters.clconv_base64
@@ -377,6 +412,20 @@ apply_local_docroot:
   - :direct
   - :sdk
   :cltype: :opt_without_arg
+src_base:
+  :desc: |-
+    Specify the prefix to be stripped off from each source object.
+    The remaining portion of the source path is kept intact at the destination.
+    Special care must be taken when used with cloud storage.
+
+
+  :tragents:
+  - :direct
+  - :node
+  - :sdk
+  :cltype: :opt_with_arg
+  :clswitch: "--src-base64"
+  :clconvert: Aspera::Fasp::Parameters.clconv_base64
 preserve_acls:
   :desc: "Preserve access control lists."
   :enum:
@@ -440,14 +489,17 @@ remove_empty_source_directory:
   - :sdk
   :cltype: :opt_without_arg
 EX_at_rest_password:
-  :desc: "Passphrase used for at rest encryption or decryption. Prefer to use standard: content_protection_password"
+  :desc: "DEPRECATED: Prefer to use standard parameter: content_protection_password"
   :tragents:
   - :direct
   :cltype: :envvar
   :clvarname: ASPERA_SCP_FILEPASS
 EX_proxy_password:
-  :desc: "Password used for Aspera proxy server authentication.\n
-May be overridden by password in URL EX_fasp_proxy_url."
+  :desc: |-
+    Password used for Aspera proxy server authentication.
+    May be overridden by password in URL EX_fasp_proxy_url.
+
+
   :tragents:
   - :direct
   :cltype: :envvar

data/lib/aspera/fasp/resume_policy.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'singleton'
 require 'aspera/log'
 
@@ -7,22 +8,22 @@ module Aspera
     # implements a simple resume policy
     class ResumePolicy
       # list of supported parameters and default values
-      DEFAULTS={
-        iter_max:       7,
-        sleep_initial:  2,
-        sleep_factor:   2,
-        sleep_max:      60
-      }
+      DEFAULTS = {
+        iter_max:      7,
+        sleep_initial: 2,
+        sleep_factor:  2,
+        sleep_max:     60
+      }.freeze
 
       # @param params see DEFAULTS
       def initialize(params=nil)
-        @parameters=DEFAULTS.clone
+        @parameters = DEFAULTS.dup
         if !params.nil?
           raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
           params.each do |k,v|
             raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.has_key?(k)
             raise "#{k} must be Integer" unless v.is_a?(Integer)
-            @parameters[k]=v
+            @parameters[k] = v
           end
         end
         Log.log.debug("resume params=#{@parameters}")
@@ -30,7 +31,8 @@ module Aspera
 
       # calls block a number of times (resumes) until success or limit reached
       # this is re-entrant, one resumer can handle multiple transfers in //
-      def process(&block)
+      def execute_with_resume
+        raise 'block manndatory' unless block_given?
         # maximum of retry
         remaining_resumes = @parameters[:iter_max]
         sleep_seconds = @parameters[:sleep_initial]
@@ -39,7 +41,8 @@ module Aspera
         loop do
           Log.log.debug('transfer starting');
           begin
-            block.call
+            # call provided block
+            yield
             break
           rescue Fasp::Error => e
             Log.log.warn("An error occured: #{e.message}");
@@ -57,7 +60,7 @@ module Aspera
           end
 
           # take this retry in account
-          remaining_resumes-=1
+          remaining_resumes -= 1
           Log.log.warn("resuming in  #{sleep_seconds} seconds (retry left:#{remaining_resumes})");
 
           # wait a bit before retrying, maybe network condition will be better

data/lib/aspera/fasp/transfer_spec.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/parameters'
 
 module Aspera
@@ -6,14 +7,14 @@ module Aspera
     # parameters for Transfer Spec
     class TransferSpec
       # default transfer username for access key based transfers
-      ACCESS_KEY_TRANSFER_USER='xfer'
-      SSH_PORT=33_001
-      UDP_PORT=33_001
-      AK_TSPEC_BASE={
+      ACCESS_KEY_TRANSFER_USER = 'xfer'
+      SSH_PORT = 33_001
+      UDP_PORT = 33_001
+      AK_TSPEC_BASE = {
         'remote_user' => ACCESS_KEY_TRANSFER_USER,
         'ssh_port'    => SSH_PORT,
         'fasp_port'   => UDP_PORT
-      }
+      }.freeze
       # define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
       Aspera::Fasp::Parameters.description.each do |k,v|
         next unless v[:enum].is_a?(Array)

data/lib/aspera/fasp/uri.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/log'
 require 'aspera/command_line_builder'
 
@@ -7,38 +8,38 @@ module Aspera
     # translates a "faspe:" URI (used in Faspex) into transfer spec hash
     class Uri
       def initialize(fasplink)
-        @fasp_uri=URI.parse(fasplink.gsub(' ','%20'))
+        @fasp_uri = URI.parse(fasplink.gsub(' ','%20'))
         # TODO: check scheme is faspe
       end
 
       def transfer_spec
-        result_ts={}
-        result_ts['remote_host']=@fasp_uri.host
-        result_ts['remote_user']=@fasp_uri.user
-        result_ts['ssh_port']=@fasp_uri.port
-        result_ts['paths']=[{'source'=>URI.decode_www_form_component(@fasp_uri.path)}]
+        result_ts = {}
+        result_ts['remote_host'] = @fasp_uri.host
+        result_ts['remote_user'] = @fasp_uri.user
+        result_ts['ssh_port'] = @fasp_uri.port
+        result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
         # faspex does not encode trailing base64 encoded tags, fix that
-        fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D'*x.length}
+        fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D' * x.length}
 
         URI.decode_www_form(fixed_query).each do |i|
-          name=i[0]
-          value=i[1]
+          name = i[0]
+          value = i[1]
           case name
-          when 'cookie' then result_ts['cookie']=value
-          when 'token' then result_ts['token']=value
-          when 'sshfp' then result_ts['sshfp']=value
-          when 'policy' then result_ts['rate_policy']=value
-          when 'httpport' then result_ts['http_fallback_port']=value.to_i
-          when 'targetrate' then result_ts['target_rate_kbps']=value.to_i
-          when 'minrate' then result_ts['min_rate_kbps']=value.to_i
-          when 'port' then result_ts['fasp_port']=value.to_i
-          when 'bwcap' then result_ts['target_rate_cap_kbps']=value.to_i
-          when 'enc' then result_ts['cipher']=value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
-          when 'tags64' then result_ts['tags']=JSON.parse(Base64.strict_decode64(value))
-          when 'createpath' then result_ts['create_dir']=CommandLineBuilder.yes_to_true(value)
-          when 'fallback' then result_ts['http_fallback']=CommandLineBuilder.yes_to_true(value)
-          when 'lockpolicy' then result_ts['lock_rate_policy']=CommandLineBuilder.yes_to_true(value)
-          when 'lockminrate' then result_ts['lock_min_rate']=CommandLineBuilder.yes_to_true(value)
+          when 'cookie' then result_ts['cookie'] = value
+          when 'token' then result_ts['token'] = value
+          when 'sshfp' then result_ts['sshfp'] = value
+          when 'policy' then result_ts['rate_policy'] = value
+          when 'httpport' then result_ts['http_fallback_port'] = value.to_i
+          when 'targetrate' then result_ts['target_rate_kbps'] = value.to_i
+          when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
+          when 'port' then result_ts['fasp_port'] = value.to_i
+          when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
+          when 'enc' then result_ts['cipher'] = value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
+          when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
+          when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
+          when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
+          when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
+          when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
           when 'auth' then Log.log.debug("ignoring auth #{name}=#{value}") # TODO: translate into transfer spec ? yes/no
           when 'v' then Log.log.debug("ignoring v #{name}=#{value}") # TODO: translate into transfer spec ? 2
           when 'protect' then Log.log.debug("ignoring protect #{name}=#{value}") # TODO: translate into transfer spec ?