aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/fasp/parameters.rb

@@ -4,105 +4,82 @@ require 'aspera/temp_file_manager'
 require 'securerandom'
 require 'base64'
 require 'json'
+require 'yaml'
 require 'securerandom'
 require 'fileutils'
+require 'openssl'
 
 module Aspera
   module Fasp
     # translate transfer specification to ascp parameter list
     class Parameters
       private
-      # temp folder for file lists, must contain only file lists
+      # Temp folder for file lists, must contain only file lists
       # because of garbage collection takes any file there
       # this could be refined, as , for instance, on macos, temp folder is already user specific
       @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-      PARAM_DEFINITION={
-        # parameters with env vars
-        'remote_password'         => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
-        'token'                   => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
-        'cookie'                  => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
-        'ssh_private_key'         => { :type => :envvar, :variable=>'ASPERA_SCP_KEY'},
-        'EX_at_rest_password'     => { :type => :envvar, :variable=>'ASPERA_SCP_FILEPASS'},
-        'EX_proxy_password'       => { :type => :envvar, :variable=>'ASPERA_PROXY_PASS'},
-        'EX_license_text'         => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
-        # bool params
-        'create_dir'              => { :type => :opt_without_arg, :option_switch=>'-d'},
-        'precalculate_job_size'   => { :type => :opt_without_arg},
-        'keepalive'               => { :type => :opt_without_arg},
-        'delete_before_transfer'  => { :type => :opt_without_arg}, #TODO: doc readme
-        'preserve_access_time'    => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_creation_time'  => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_times'          => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_modification_time'=> { :type => :opt_without_arg}, #TODO: doc
-        'remove_empty_directories'=> { :type => :opt_without_arg}, #TODO: doc
-        'remove_after_transfer'   => { :type => :opt_without_arg}, #TODO: doc
-        'remove_empty_source_directory'=> { :type => :opt_without_arg}, #TODO: doc
-        # value params
-        'cipher'                  => { :type => :opt_with_arg, :option_switch=>'-c',:accepted_types=>String,:encode=>lambda{|cipher|cipher.tr('-','')}},
-        'resume_policy'           => { :type => :opt_with_arg, :option_switch=>'-k',:accepted_types=>String,:default=>'sparse_csum',:translate_values=>{'none'=>0,'attrs'=>1,'sparse_csum'=>2,'full_csum'=>3}},
-        'direction'               => { :type => :opt_with_arg, :option_switch=>'--mode',:accepted_types=>String,:translate_values=>{'receive'=>'recv','send'=>'send'}},
-        'remote_user'             => { :type => :opt_with_arg, :option_switch=>'--user',:accepted_types=>String},
-        'remote_host'             => { :type => :opt_with_arg, :option_switch=>'--host',:accepted_types=>String},
-        'ssh_port'                => { :type => :opt_with_arg, :option_switch=>'-P',:accepted_types=>Integer},
-        'fasp_port'               => { :type => :opt_with_arg, :option_switch=>'-O',:accepted_types=>Integer},
-        'dgram_size'              => { :type => :opt_with_arg, :option_switch=>'-Z',:accepted_types=>Integer},
-        'target_rate_kbps'        => { :type => :opt_with_arg, :option_switch=>'-l',:accepted_types=>Integer},
-        'min_rate_kbps'           => { :type => :opt_with_arg, :option_switch=>'-m',:accepted_types=>Integer},
-        'rate_policy'             => { :type => :opt_with_arg, :option_switch=>'--policy',:accepted_types=>String},
-        'http_fallback'           => { :type => :opt_with_arg, :option_switch=>'-y',:accepted_types=>[String,*Aspera::CommandLineBuilder::BOOLEAN_CLASSES],:translate_values=>{'force'=>'F',true=>1,false=>0}},
-        'http_fallback_port'      => { :type => :opt_with_arg, :option_switch=>'-t',:accepted_types=>Integer},
-        'source_root'             => { :type => :opt_with_arg, :option_switch=>'--source-prefix64',:accepted_types=>String,:encode=>lambda{|prefix|Base64.strict_encode64(prefix)}},
-        'sshfp'                   => { :type => :opt_with_arg, :option_switch=>'--check-sshfp',:accepted_types=>String},
-        'symlink_policy'          => { :type => :opt_with_arg, :option_switch=>'--symbolic-links',:accepted_types=>String},
-        'overwrite'               => { :type => :opt_with_arg, :accepted_types=>String},
-        'exclude_newer_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
-        'exclude_older_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
-        'preserve_acls'           => { :type => :opt_with_arg, :accepted_types=>String},
-        'move_after_transfer'     => { :type => :opt_with_arg, :accepted_types=>String},
-        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>Integer},
-        # non standard parameters
-        'EX_fasp_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
-        'EX_http_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
-        'EX_ssh_key_paths'        => { :type => :opt_with_arg, :option_switch=>'-i',:accepted_types=>Array},
-        'EX_http_transfer_jpeg'   => { :type => :opt_with_arg, :option_switch=>'-j',:accepted_types=>Integer},
-        'EX_multi_session_part'   => { :type => :opt_with_arg, :option_switch=>'-C',:accepted_types=>String},
-        'EX_no_read'              => { :type => :opt_without_arg, :option_switch=>'--no-read'},
-        'EX_no_write'             => { :type => :opt_without_arg, :option_switch=>'--no-write'},
-        'EX_apply_local_docroot'  => { :type => :opt_without_arg, :option_switch=>'--apply-local-docroot'},
-        # TODO: manage those parameters, some are for connect only ? node api ?
-        'target_rate_cap_kbps'    => { :type => :ignore, :accepted_types=>Integer},
-        'target_rate_percentage'  => { :type => :ignore, :accepted_types=>String}, # -wf -l<rate>p
-        'min_rate_cap_kbps'       => { :type => :ignore, :accepted_types=>Integer},
-        'rate_policy_allowed'     => { :type => :ignore, :accepted_types=>String},
-        'fasp_url'                => { :type => :ignore, :accepted_types=>String},
-        'lock_rate_policy'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'lock_min_rate'           => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'lock_target_rate'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'authentication'          => { :type => :ignore, :accepted_types=>String}, # value = token
-        'https_fallback_port'     => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
-        'content_protection'      => { :type => :ignore, :accepted_types=>String},
-        'cipher_allowed'          => { :type => :ignore, :accepted_types=>String},
-        'multi_session'           => { :type => :ignore, :accepted_types=>Integer}, # managed
-        'obfuscate_file_names'          => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        # optional tags (  additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'}  )
-        'tags'                    => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
-        # special processing @builder.process_param( called individually
-        'use_ascp4'               => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'paths'                   => { :type => :defer, :accepted_types=>Array},
-        'EX_file_list'            => { :type => :defer, :option_switch=>'--file-list', :accepted_types=>String},
-        'EX_file_pair_list'       => { :type => :defer, :option_switch=>'--file-pair-list', :accepted_types=>String},
-        'EX_ascp_args'            => { :type => :defer, :accepted_types=>Array},
-        'destination_root'        => { :type => :defer, :accepted_types=>String},
-        'wss_enabled'             => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'wss_port'                => { :type => :defer, :accepted_types=>Integer},
-      }
-
-      private_constant :PARAM_DEFINITION
+      @@param_description_cache=nil
+      # @return normaiwed description of transfer spec parameters
+      def self.description
+        return @@param_description_cache unless @@param_description_cache.nil?
+        # config file in same folder with same name as this source
+        @@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
+        Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
+      end
+
+      # Agents shown in manual for parameters (sub list)
+      SUPPORTED_AGENTS=[:direct,:node,:connect]
+      # Short names of columns in manual
+      SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
+
+      # @return a table suitable to display a manual
+      def self.man_table
+        result=[]
+        description.keys.map do |k|
+          i=description[k]
+          param={name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
+          SUPPORTED_AGENTS.each do |a|
+            param[a.to_s[0].to_sym]=i[:context].nil? || i[:context].include?(a) ? 'Y' : ''
+          end
+          # only keep lines that are usable in supported agents
+          next if SUPPORTED_AGENTS_SHORT.inject(true){|m,i|m and param[i].empty?}
+          param[:cli]=case i[:cltype]
+          when :envvar; 'env:'+i[:clvarname]
+          when :opt_without_arg,:opt_with_arg; i[:option_switch]
+          else ''
+          end
+          if i.has_key?(:enum)
+            param[:description] << "\nAllowed values: #{i[:enum].join(', ')}"
+          end
+          result.push(param)
+        end
+        return result
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_cipher(v)
+        v.tr('-','')
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_source_root(v)
+        Base64.strict_encode64(v)
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_tags(v)
+        Base64.strict_encode64(JSON.generate(v))
+      end
+
+      def self.ts_has_file_list(ts)
+        ts.has_key?('EX_ascp_args') and ts['EX_ascp_args'].is_a?(Array) and ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}
+      end
 
       def initialize(job_spec,options)
         @job_spec=job_spec
-        @builder=Aspera::CommandLineBuilder.new(@job_spec,PARAM_DEFINITION)
         @options=options
+        @builder=Aspera::CommandLineBuilder.new(@job_spec,self.class.description)
+        Log.log.debug("agent options: #{@options}")
       end
 
       public
@@ -125,8 +102,8 @@ module Aspera
         # special cases
         @job_spec.delete('source_root') if @job_spec.has_key?('source_root') and @job_spec['source_root'].empty?
 
-        # use web socket initiation ?
-        if @builder.process_param('wss_enabled',:get_value) and @options[:wss]
+        # use web socket session initiation ?
+        if @builder.process_param('wss_enabled',:get_value) and ( @options[:wss] or !@job_spec.has_key?('fasp_port') )
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
           # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
@@ -135,6 +112,9 @@ module Aspera
           @job_spec.delete('fasp_port')
           @job_spec.delete('EX_ssh_key_paths')
           @job_spec.delete('sshfp')
+          # set default location for CA bundle, see env var SSL_CERT_FILE / SSL_CERT_DIR
+          @job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
+          Log.log.debug("CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl")
         else
           # remove unused parameter (avoid warning)
           @job_spec.delete('wss_port')
@@ -151,10 +131,17 @@ module Aspera
           # destination will be base64 encoded, put before path arguments
           @builder.add_command_line_options(['--dest64'])
         end
-
-        PARAM_DEFINITION['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
+        # paths is mandatory, unless ...
+        file_list_provided=self.class.ts_has_file_list(@job_spec)
+        @builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive') and !file_list_provided
         paths_array=@builder.process_param('paths',:get_value)
-        unless paths_array.nil?
+        if file_list_provided and ! paths_array.nil?
+          Log.log.warn("file list provided both in transfer spec and ascp file list. Keeping file list only.")
+          paths_array=nil
+        end
+        if ! paths_array.nil?
+          # it's an array
+          raise "paths is empty in transfer spec" if paths_array.empty?
           # use file list if there is storage defined for it.
           if @@file_list_folder.nil?
             # not safe for special characters ? (maybe not, depends on OS)
@@ -180,8 +167,8 @@ module Aspera
                   lines=paths_array.map{|i|i['source']}
                 end
                 file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
-                File.open(file_list_file, 'w+'){|f|f.puts(lines)}
-                Log.log.debug("#{option}=\n#{File.read(file_list_file)}".red)
+                File.open(file_list_file, 'w+'){|f|f.write(lines.join("\n"))}
+                Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
               end
             end
             @builder.add_command_line_options(["#{option}=#{file_list_file}"])