aspera-cli 4.2.1 → 4.2.2

data/lib/aspera/cli/transfer_agent.rb

@@ -16,33 +16,47 @@ module Aspera
       FILE_LIST_FROM_ARGS='@args'
       # special value for --sources : read file list from transfer spec (--ts)
       FILE_LIST_FROM_TRANSFER_SPEC='@ts'
-      private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC
-      # @param cli_objects external objects: option manager, config file manager
-      def initialize(cli_objects)
-        @opt_mgr=cli_objects[:options]
-        @config=cli_objects[:config]
-        # transfer spec overrides provided on command line
-        @transfer_spec_cmdline={"create_dir"=>true}
+      DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
+From: <%=from_name%> <<%=from_email%>>
+To: <<%=to%>>
+Subject: <%=subject%>
+
+Transfer is: <%=global_transfer_status%>
+
+<%=ts.to_yaml%>
+END_OF_TEMPLATE
+      #%
+      private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
+      # @param env external objects: option manager, config file manager
+      def initialize(env)
+        # same as plugin environment
+        @env=env
+        # command line can override transfer spec
+        @transfer_spec_cmdline={'create_dir'=>true}
         # the currently selected transfer agent
         @agent=nil
         @progress_listener=Listener::ProgressMulti.new
         # source/destination pair, like "paths" of transfer spec
         @transfer_paths=nil
-        @opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
-        @opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
-        @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
-        @opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
-        @opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
-        @opt_mgr.add_opt_simple(:transfer_info,"additional information for transfer client")
-        @opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
-        @opt_mgr.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
-        @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
-        @opt_mgr.set_option(:transfer,:direct)
-        @opt_mgr.set_option(:src_type,:list)
-        @opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
-        @opt_mgr.parse_options!
+        options.set_obj_attr(:ts,self,:option_transfer_spec)
+        options.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{options.get_option(:ts,:optional)}")
+        options.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{options.get_option(:local_resume,:optional)}")
+        options.add_opt_simple(:to_folder,"destination folder for downloaded files")
+        options.add_opt_simple(:sources,"list of source files (see doc)")
+        options.add_opt_simple(:transfer_info,"additional information for transfer client")
+        options.add_opt_list(:src_type,[:list,:pair],"type of file list")
+        options.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
+        options.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
+        options.set_option(:transfer,:direct)
+        options.set_option(:src_type,:list)
+        options.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
+        options.parse_options!
       end
 
+      def options; @env[:options];end
+
+      def config; @env[:config];end
+
       def option_transfer_spec; @transfer_spec_cmdline; end
 
       # multiple option are merged
@@ -54,8 +68,8 @@ module Aspera
         @agent=instance
         @agent.add_listener(Listener::Logger.new)
         # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
-        if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
-        (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and !@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct))
+        if options.get_option(:progress,:mandatory).eql?(:multi) or
+        (options.get_option(:progress,:mandatory).eql?(:native) and !options.get_option(:transfer,:mandatory).eql?(:direct))
           @agent.add_listener(@progress_listener)
         end
       end
@@ -63,15 +77,15 @@ module Aspera
       # analyze options and create new agent if not already created or set
       def set_agent_by_options
         return nil unless @agent.nil?
-        agent_type=@opt_mgr.get_option(:transfer,:mandatory)
+        agent_type=options.get_option(:transfer,:mandatory)
         case agent_type
         when :direct
-          agent_options=@opt_mgr.get_option(:transfer_info,:optional)
+          agent_options=options.get_option(:transfer_info,:optional)
           agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
           new_agent=Fasp::Local.new(agent_options)
-          new_agent.quiet=false if @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
+          new_agent.quiet=false if options.get_option(:progress,:mandatory).eql?(:native)
         when :httpgw
-          httpgw_config=@opt_mgr.get_option(:transfer_info,:mandatory)
+          httpgw_config=options.get_option(:transfer_info,:mandatory)
           new_agent=Fasp::HttpGW.new(httpgw_config)
         when :connect
           new_agent=Fasp::Connect.new
@@ -79,12 +93,12 @@ module Aspera
           # way for code to setup alternate node api in advance
           # support: @preset:<name>
           # support extended values
-          node_config=@opt_mgr.get_option(:transfer_info,:optional)
+          node_config=options.get_option(:transfer_info,:optional)
           # if not specified: use default node
           if node_config.nil?
-            param_set_name=@config.get_plugin_default_config_name(:node)
+            param_set_name=config.get_plugin_default_config_name(:node)
             raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
-            node_config=@config.preset_by_name(param_set_name)
+            node_config=config.preset_by_name(param_set_name)
           end
           Log.log.debug("node=#{node_config}")
           raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
@@ -103,11 +117,11 @@ module Aspera
             }})
           new_agent=Fasp::Node.new(node_api)
         when :aoc
-          aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
+          aoc_config=options.get_option(:transfer_info,:optional)
           if aoc_config.nil?
-            param_set_name=@config.get_plugin_default_config_name(:aspera)
+            param_set_name=config.get_plugin_default_config_name(:aspera)
             raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
-            aoc_config=@config.preset_by_name(param_set_name)
+            aoc_config=config.preset_by_name(param_set_name)
           end
           Log.log.debug("aoc=#{aoc_config}")
           raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
@@ -129,7 +143,7 @@ module Aspera
       # sets default if needed
       # param: 'send' or 'receive'
       def destination_folder(direction)
-        dest_folder=@opt_mgr.get_option(:to_folder,:optional)
+        dest_folder=options.get_option(:to_folder,:optional)
         return dest_folder unless dest_folder.nil?
         dest_folder=@transfer_spec_cmdline['destination_root']
         return dest_folder unless dest_folder.nil?
@@ -152,12 +166,12 @@ module Aspera
         # start with lower priority : get paths from transfer spec on command line
         @transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
         # is there a source list option ?
-        file_list=@opt_mgr.get_option(:sources,:optional)
+        file_list=options.get_option(:sources,:optional)
         case file_list
         when nil,FILE_LIST_FROM_ARGS
           Log.log.debug("getting file list as parameters")
           # get remaining arguments
-          file_list=@opt_mgr.get_next_argument("source file list",:multiple)
+          file_list=options.get_next_argument("source file list",:multiple)
           raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
         when FILE_LIST_FROM_TRANSFER_SPEC
           Log.log.debug("assume list provided in transfer spec")
@@ -174,7 +188,7 @@ module Aspera
         if !@transfer_paths.nil?
           Log.log.warn("--sources overrides paths from --ts")
         end
-        case @opt_mgr.get_option(:src_type,:mandatory)
+        case options.get_option(:src_type,:mandatory)
         when :list
           # when providing a list, just specify source
           @transfer_paths=file_list.map{|i|{'source'=>i}}
@@ -189,20 +203,20 @@ module Aspera
 
       # start a transfer and wait for completion, plugins shall use this method
       # @param transfer_spec
-      # @param options specific options for the transfer_agent
-      # options[:src] specifies how destination_root is set (how transfer spec was generated)
+      # @param tr_opts specific options for the transfer_agent
+      # tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
       # other options are carried to specific agent
-      def start(transfer_spec,options)
+      def start(transfer_spec,tr_opts)
         # check parameters
         raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
-        raise "options must be hash" unless options.is_a?(Hash)
+        raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
         # process :src option
         case transfer_spec['direction']
         when 'receive'
           # init default if required in any case
           @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
         when 'send'
-          case options[:src]
+          case tr_opts[:src]
           when :direct
             # init default if required
             @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
@@ -213,12 +227,12 @@ module Aspera
           when :node_gen4
             @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
           else
-            raise StandardError,"InternalError: unsupported value: #{options[:src]}"
+            raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
           end
         end
 
         # only used here
-        options.delete(:src)
+        tr_opts.delete(:src)
 
         # update command line paths, unless destination already has one
         @transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
@@ -227,13 +241,26 @@ module Aspera
         # create transfer agent
         self.set_agent_by_options
         Log.log.debug("transfer agent is a #{@agent.class}")
-        @agent.start_transfer(transfer_spec,options)
+        @agent.start_transfer(transfer_spec,tr_opts)
         result=@agent.wait_for_transfers_completion
         @progress_listener.reset
         Fasp::Manager.validate_status_list(result)
+        send_email_transfer_notification(transfer_spec,result)
         return result
       end
 
+      def send_email_transfer_notification(transfer_spec,statuses)
+        return if options.get_option(:notif_to,:optional).nil?
+        global_status=self.class.session_status(statuses)
+        email_vars={
+          global_transfer_status: global_status,
+          subject: "ascli transfer: #{global_status}",
+          body: "Transfer is: #{global_status}",
+          ts: transfer_spec
+        }
+        @env[:config].send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
+      end
+
       # @return :success if all sessions statuses returned by "start" are success
       # else return the first error exception object
       def self.session_status(statuses)

data/lib/aspera/cli/version.rb

@@ -1,5 +1,5 @@
 module Aspera
   module Cli
-    VERSION = "4.2.1"
+    VERSION = "4.2.2"
   end
 end

data/lib/aspera/command_line_builder.rb

@@ -4,21 +4,46 @@ module Aspera
   # process_param is called repeatedly with all known parameters
   # add_env_args is called to get resulting param list and env var (also checks that all params were used)
   class CommandLineBuilder
+    # transform yes/no to trye/false
+    def self.yes_to_true(value)
+      case value
+      when 'yes'; return true
+      when 'no'; return false
+      end
+      raise "unsupported value: #{value}"
+    end
 
-    private
-    # default value for command line based on option name
-    def switch_name(param_name,options)
-      return options[:option_switch] if options.has_key?(:option_switch)
-      return '--'+param_name.to_s.gsub('_','-')
+    # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
+    def self.normalize_description(d)
+      d.each do |param_name,options|
+        Log.log.debug("def: #{param_name}")
+        raise "Expecting Hash, but have #{options.class} in #{param_name}" unless options.is_a?(Hash)
+        #options[:accepted_types]=:bool if options[:cltype].eql?(:envvar) and !options.has_key?(:accepted_types)
+        # by default : not mandatory
+        options[:mandatory]||=false
+        options[:desc]||=''
+        # by default : string, unless it's without arg
+        if ! options.has_key?(:accepted_types)
+          options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
+        end
+        # single type is placed in array
+        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
+        if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
+          options[:option_switch]='--'+param_name.to_s.gsub('_','-')
+        end
+      end
     end
 
+    private
+
+    # clvarname : command line variable name
     def env_name(param_name,options)
-      return options[:variable]
+      return options[:clvarname]
     end
 
     public
 
-    BOOLEAN_CLASSES=[TrueClass,FalseClass]
+    attr_reader :params_definition
 
     # @param param_hash
     def initialize(param_hash,params_definition)
@@ -44,15 +69,6 @@ module Aspera
       return nil
     end
 
-    # transform yes/no to trye/false
-    def self.yes_to_true(value)
-      case value
-      when 'yes'; return true
-      when 'no'; return false
-      end
-      raise "unsupported value: #{value}"
-    end
-
     # add options directly to ascp command line
     def add_command_line_options(options)
       return if options.nil?
@@ -71,24 +87,25 @@ module Aspera
     # @param options : options for type
     def process_param(param_name,action=nil)
       options=@params_definition[param_name]
-      action=options[:type] if action.nil?
+      action=options[:cltype] if action.nil?
       # should not happen
       raise "Internal error: ask processing of param #{param_name}" if options.nil?
-      # by default : not mandatory
-      options[:mandatory]||=false
-      if options.has_key?(:accepted_types)
-        # single type is placed in array
-        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
-      else
-        # by default : string, unless it's without arg
-        options[:accepted_types]=action.eql?(:opt_without_arg) ? BOOLEAN_CLASSES : [String]
-      end
       # check mandatory parameter (nil is valid value)
       raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
       parameter_value=@param_hash[param_name]
-      parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      #parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      expected_classes=options[:accepted_types].map do |s|
+        case s
+        when :string; String
+        when :array; Array
+        when :hash; Hash
+        when :int; Integer
+        when :bool; [TrueClass,FalseClass]
+        else raise "INTERNAL: unexpected value: #{s}"
+        end
+      end.flatten
       # check provided type
-      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or options[:accepted_types].inject(false){|m,v|m or parameter_value.is_a?(v)}
+      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
       @used_param_names.push(param_name) unless action.eql?(:defer)
 
       # process only non-nil values
@@ -102,7 +119,8 @@ module Aspera
       end
       raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
       if options[:encode]
-        newvalue=options[:encode].call(parameter_value)
+        # :encode has name of class with encoding method
+        newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
         raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
         parameter_value=newvalue
       end
@@ -123,12 +141,12 @@ module Aspera
         else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
         end
         add_param=!add_param if options[:add_on_false]
-        add_command_line_options([switch_name(param_name,options)]) if add_param
+        add_command_line_options([options[:option_switch]]) if add_param
       when :opt_with_arg # transform into command line option with value
         #parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
         parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
         # if transfer_spec value is an array, applies option many times
-        parameter_value.each{|v|add_command_line_options([switch_name(param_name,options),v])}
+        parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
       else
         raise "Error"
       end

data/lib/aspera/fasp/parameters.rb

@@ -4,6 +4,7 @@ require 'aspera/temp_file_manager'
 require 'securerandom'
 require 'base64'
 require 'json'
+require 'yaml'
 require 'securerandom'
 require 'fileutils'
 
@@ -12,97 +13,67 @@ module Aspera
     # translate transfer specification to ascp parameter list
     class Parameters
       private
-      # temp folder for file lists, must contain only file lists
+      # Temp folder for file lists, must contain only file lists
       # because of garbage collection takes any file there
       # this could be refined, as , for instance, on macos, temp folder is already user specific
       @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-      PARAM_DEFINITION={
-        # parameters with env vars
-        'remote_password'         => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
-        'token'                   => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
-        'cookie'                  => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
-        'ssh_private_key'         => { :type => :envvar, :variable=>'ASPERA_SCP_KEY'},
-        'EX_at_rest_password'     => { :type => :envvar, :variable=>'ASPERA_SCP_FILEPASS'},
-        'EX_proxy_password'       => { :type => :envvar, :variable=>'ASPERA_PROXY_PASS'},
-        'EX_license_text'         => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
-        # bool params
-        'create_dir'              => { :type => :opt_without_arg, :option_switch=>'-d'},
-        'precalculate_job_size'   => { :type => :opt_without_arg},
-        'keepalive'               => { :type => :opt_without_arg},
-        'delete_before_transfer'  => { :type => :opt_without_arg}, #TODO: doc readme
-        'preserve_access_time'    => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_creation_time'  => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_times'          => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_modification_time'=> { :type => :opt_without_arg}, #TODO: doc
-        'remove_empty_directories'=> { :type => :opt_without_arg}, #TODO: doc
-        'remove_after_transfer'   => { :type => :opt_without_arg}, #TODO: doc
-        'remove_empty_source_directory'=> { :type => :opt_without_arg}, #TODO: doc
-        # value params
-        'cipher'                  => { :type => :opt_with_arg, :option_switch=>'-c',:accepted_types=>String,:encode=>lambda{|cipher|cipher.tr('-','')}},
-        'resume_policy'           => { :type => :opt_with_arg, :option_switch=>'-k',:accepted_types=>String,:default=>'sparse_csum',:translate_values=>{'none'=>0,'attrs'=>1,'sparse_csum'=>2,'full_csum'=>3}},
-        'direction'               => { :type => :opt_with_arg, :option_switch=>'--mode',:accepted_types=>String,:translate_values=>{'receive'=>'recv','send'=>'send'}},
-        'remote_user'             => { :type => :opt_with_arg, :option_switch=>'--user',:accepted_types=>String},
-        'remote_host'             => { :type => :opt_with_arg, :option_switch=>'--host',:accepted_types=>String},
-        'ssh_port'                => { :type => :opt_with_arg, :option_switch=>'-P',:accepted_types=>Integer},
-        'fasp_port'               => { :type => :opt_with_arg, :option_switch=>'-O',:accepted_types=>Integer},
-        'dgram_size'              => { :type => :opt_with_arg, :option_switch=>'-Z',:accepted_types=>Integer},
-        'target_rate_kbps'        => { :type => :opt_with_arg, :option_switch=>'-l',:accepted_types=>Integer},
-        'min_rate_kbps'           => { :type => :opt_with_arg, :option_switch=>'-m',:accepted_types=>Integer},
-        'rate_policy'             => { :type => :opt_with_arg, :option_switch=>'--policy',:accepted_types=>String},
-        'http_fallback'           => { :type => :opt_with_arg, :option_switch=>'-y',:accepted_types=>[String,*Aspera::CommandLineBuilder::BOOLEAN_CLASSES],:translate_values=>{'force'=>'F',true=>1,false=>0}},
-        'http_fallback_port'      => { :type => :opt_with_arg, :option_switch=>'-t',:accepted_types=>Integer},
-        'source_root'             => { :type => :opt_with_arg, :option_switch=>'--source-prefix64',:accepted_types=>String,:encode=>lambda{|prefix|Base64.strict_encode64(prefix)}},
-        'sshfp'                   => { :type => :opt_with_arg, :option_switch=>'--check-sshfp',:accepted_types=>String},
-        'symlink_policy'          => { :type => :opt_with_arg, :option_switch=>'--symbolic-links',:accepted_types=>String},
-        'overwrite'               => { :type => :opt_with_arg, :accepted_types=>String},
-        'exclude_newer_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
-        'exclude_older_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
-        'preserve_acls'           => { :type => :opt_with_arg, :accepted_types=>String},
-        'move_after_transfer'     => { :type => :opt_with_arg, :accepted_types=>String},
-        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>Integer},
-        # non standard parameters
-        'EX_fasp_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
-        'EX_http_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
-        'EX_ssh_key_paths'        => { :type => :opt_with_arg, :option_switch=>'-i',:accepted_types=>Array},
-        'EX_http_transfer_jpeg'   => { :type => :opt_with_arg, :option_switch=>'-j',:accepted_types=>Integer},
-        'EX_multi_session_part'   => { :type => :opt_with_arg, :option_switch=>'-C',:accepted_types=>String},
-        'EX_no_read'              => { :type => :opt_without_arg, :option_switch=>'--no-read'},
-        'EX_no_write'             => { :type => :opt_without_arg, :option_switch=>'--no-write'},
-        'EX_apply_local_docroot'  => { :type => :opt_without_arg, :option_switch=>'--apply-local-docroot'},
-        # TODO: manage those parameters, some are for connect only ? node api ?
-        'target_rate_cap_kbps'    => { :type => :ignore, :accepted_types=>Integer},
-        'target_rate_percentage'  => { :type => :ignore, :accepted_types=>String}, # -wf -l<rate>p
-        'min_rate_cap_kbps'       => { :type => :ignore, :accepted_types=>Integer},
-        'rate_policy_allowed'     => { :type => :ignore, :accepted_types=>String},
-        'fasp_url'                => { :type => :ignore, :accepted_types=>String},
-        'lock_rate_policy'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'lock_min_rate'           => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'lock_target_rate'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'authentication'          => { :type => :ignore, :accepted_types=>String}, # value = token
-        'https_fallback_port'     => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
-        'content_protection'      => { :type => :ignore, :accepted_types=>String},
-        'cipher_allowed'          => { :type => :ignore, :accepted_types=>String},
-        'multi_session'           => { :type => :ignore, :accepted_types=>Integer}, # managed
-        'obfuscate_file_names'          => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        # optional tags (  additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'}  )
-        'tags'                    => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
-        # special processing @builder.process_param( called individually
-        'use_ascp4'               => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'paths'                   => { :type => :defer, :accepted_types=>Array},
-        'EX_file_list'            => { :type => :defer, :option_switch=>'--file-list', :accepted_types=>String},
-        'EX_file_pair_list'       => { :type => :defer, :option_switch=>'--file-pair-list', :accepted_types=>String},
-        'EX_ascp_args'            => { :type => :defer, :accepted_types=>Array},
-        'destination_root'        => { :type => :defer, :accepted_types=>String},
-        'wss_enabled'             => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'wss_port'                => { :type => :defer, :accepted_types=>Integer},
-      }
-
-      private_constant :PARAM_DEFINITION
+      @@param_description_cache=nil
+      # @return normaiwed description of transfer spec parameters
+      def self.description
+        return @@param_description_cache unless @@param_description_cache.nil?
+        # config file in same folder with same name as this source
+        @@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
+        Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
+      end
+
+      # Agents shown in manual
+      SUPPORTED_AGENTS=[:direct,:node,:connect]
+      # Short names of columns in manual
+      SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
+
+      # @return a table suitable to display a manual
+      def self.man_table
+        result=[]
+        description.keys.map do |k|
+          i=description[k]
+          param={name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
+          SUPPORTED_AGENTS.each do |a|
+            param[a.to_s[0].to_sym]=i[:context].nil? || i[:context].include?(a) ? 'Y' : ''
+          end
+          # only keep lines that are usable in supported agents
+          next if SUPPORTED_AGENTS_SHORT.inject(true){|m,i|m and param[i].empty?}
+          param[:cli]=case i[:cltype]
+          when :envvar; 'env:'+i[:clvarname]
+          when :opt_without_arg,:opt_with_arg; i[:option_switch]
+          else ''
+          end
+          if i.has_key?(:enum)
+            param[:description] << "\nAllowed values: #{i[:enum].join(', ')}"
+          end
+          result.push(param)
+        end
+        return result
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_cipher(v)
+        v.tr('-','')
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_source_root(v)
+        Base64.strict_encode64(v)
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_tags(v)
+        Base64.strict_encode64(JSON.generate(v))
+      end
 
       def initialize(job_spec,options)
         @job_spec=job_spec
-        @builder=Aspera::CommandLineBuilder.new(@job_spec,PARAM_DEFINITION)
         @options=options
+        @builder=Aspera::CommandLineBuilder.new(@job_spec,self.class.description)
       end
 
       public
@@ -125,7 +96,7 @@ module Aspera
         # special cases
         @job_spec.delete('source_root') if @job_spec.has_key?('source_root') and @job_spec['source_root'].empty?
 
-        # use web socket initiation ?
+        # use web socket session initiation ?
         if @builder.process_param('wss_enabled',:get_value) and @options[:wss]
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
@@ -151,8 +122,7 @@ module Aspera
           # destination will be base64 encoded, put before path arguments
           @builder.add_command_line_options(['--dest64'])
         end
-
-        PARAM_DEFINITION['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
+        @builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
         paths_array=@builder.process_param('paths',:get_value)
         unless paths_array.nil?
           # use file list if there is storage defined for it.