aspera-cli 4.1.0 → 4.3.0

data/lib/aspera/fasp/local.rb

@@ -21,16 +21,25 @@ module Aspera
     ACCESS_KEY_TRANSFER_USER='xfer'
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class Local < Manager
-      ASCP_SPAWN_TIMEOUT_SEC = 3
-      private_constant :ASCP_SPAWN_TIMEOUT_SEC
-      # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
+      # options for initialize (same as values in option transfer_info)
+      DEFAULT_OPTIONS = {
+        :spawn_timeout_sec => 3,
+        :spawn_delay_sec   => 2,
+        :wss               => false,
+        :multi_incr_udp    => true,
+        :resume            => {}
+      }
+      DEFAULT_UDP_PORT=33001
+      private_constant :DEFAULT_OPTIONS
+      # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
       attr_accessor :quiet
+
       # start ascp transfer (non blocking), single or multi-session
       # job information added to @jobs
       # @param transfer_spec [Hash] aspera transfer specification
       # @param options [Hash] :resumer, :regenerate_token
       def start_transfer(transfer_spec,options={})
-        raise "option: must be hash (or nil)" unless options.is_a?(Hash)
+        raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
         job_options = options.clone
         job_options[:resumer] ||= @resume_policy
         job_options[:job_id] ||= SecureRandom.uuid
@@ -56,23 +65,34 @@ module Aspera
           transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
         end
 
-        # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
-        # compute this before using transfer spec, even if the var is not used in single session
-        multi_session_udp_port_base=33001
-        multi_session_number=nil
+        # Compute this before using transfer spec because it potentially modifies the transfer spec
+        # (even if the var is not used in single session)
+        multi_session_info=nil
         if transfer_spec.has_key?('multi_session')
-          multi_session_number=transfer_spec['multi_session'].to_i
-          raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
-          # managed here, so delete from transfer spec
+          multi_session_info={
+            count: transfer_spec['multi_session'].to_i,
+          }
+          # Managed by multi-session, so delete from transfer spec
           transfer_spec.delete('multi_session')
-          if transfer_spec.has_key?('fasp_port')
-            multi_session_udp_port_base=transfer_spec['fasp_port']
-            transfer_spec.delete('fasp_port')
+          if multi_session_info[:count] < 0
+            Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
+            multi_session_info = nil
+          elsif multi_session_info[:count].eql?(0)
+            Log.log.debug("multi_session  count is zero: no multisession")
+            multi_session_info = nil
+          else # multi_session_info[:count] > 0
+            # if option not true: keep default udp port for all sessions
+            if @options[:multi_incr_udp]
+              # override if specified, else use default value
+              multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : DEFAULT_UDP_PORT
+              # delete from original transfer spec, as we will increment values
+              transfer_spec.delete('fasp_port')
+            end
           end
         end
 
         # compute known args
-        env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
+        env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
 
         # add fallback cert and key as arguments if needed
         if ['1','force'].include?(transfer_spec['http_fallback'])
@@ -98,25 +118,28 @@ module Aspera
           :options  => job_options  # [Hash]
         }
 
-        Log.log.debug("starting session thread(s)")
-        if !multi_session_number
+        if multi_session_info.nil?
+          Log.log.debug('Starting single session thread')
           # single session for transfer : simple
           session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
           xfer_job[:sessions].push(session)
         else
-          1.upto(multi_session_number) do |i|
+          Log.log.debug('Starting multi session threads')
+          1.upto(multi_session_info[:count]) do |i|
+            # do not delay the first session
+            sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session=session.clone()
             this_session[:env_args]=this_session[:env_args].clone()
             this_session[:env_args][:args]=this_session[:env_args][:args].clone()
-            this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
-            # necessary only if server is not linux, i.e. server does not support port re-use
-            this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
+            this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
+            # option: increment (default as per ascp manual) or not (cluster on other side ?)
+            this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
             this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
             xfer_job[:sessions].push(this_session)
           end
         end
-        Log.log.debug("started session thread(s)")
+        Log.log.debug('started session thread(s)')
 
         # add job to list of jobs
         @jobs[job_options[:job_id]]=xfer_job
@@ -128,7 +151,7 @@ module Aspera
       # wait for completion of all jobs started
       # @return list of :success or error message
       def wait_for_transfers_completion
-        Log.log.debug("wait_for_transfers_completion")
+        Log.log.debug('wait_for_transfers_completion')
         # set to non-nil to exit loop
         result=[]
         @jobs.each do |id,job|
@@ -138,7 +161,7 @@ module Aspera
             result.push(session[:error] ? session[:error] : :success)
           end
         end
-        Log.log.debug("all transfers joined")
+        Log.log.debug('all transfers joined')
         # since all are finished and we return the result, clear statuses
         @jobs.clear
         return result
@@ -146,7 +169,7 @@ module Aspera
 
       # used by asession (to be removed ?)
       def shutdown
-        Log.log.debug("fasp local shutdown")
+        Log.log.debug('fasp local shutdown')
       end
 
       # This is the low level method to start the "ascp" process
@@ -158,8 +181,10 @@ module Aspera
       # @param session this session information
       # could be private method
       def start_transfer_with_args_env(env_args,session)
-        raise "env_args must be Hash" unless env_args.is_a?(Hash)
-        raise "session must be Hash" unless session.is_a?(Hash)
+        raise 'env_args must be Hash' unless env_args.is_a?(Hash)
+        raise 'session must be Hash' unless session.is_a?(Hash)
+        # by default we assume an exception will be raised (for ensure block)
+        exception_raised=true
         begin
           Log.log.debug("env_args=#{env_args.inspect}")
           # get location of ascp executable
@@ -182,7 +207,7 @@ module Aspera
           Log.log.debug("before accept for pid (#{ascp_pid})")
           # init management socket
           ascp_mgt_io=nil
-          Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
+          Timeout.timeout(@options[:spawn_timeout_sec]) do
             ascp_mgt_io = mgt_sock.accept
             # management messages include file names which may be utf8
             # by default socket is US-ASCII
@@ -216,7 +241,7 @@ module Aspera
               current_event_data[$1] = $2
             when ''
               # empty line is separator to end event information
-              raise "unexpected empty line" if current_event_data.nil?
+              raise 'unexpected empty line' if current_event_data.nil?
               current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
               notify_listeners(current_event_text,current_event_data)
               case current_event_data['Type']
@@ -232,26 +257,28 @@ module Aspera
             end # case
           end # loop (process mgt port lines)
           # check that last status was received before process exit
-          if last_status_event.nil?
-            Log.log.warn("no status read from ascp mgt port")
-          else
+          if last_status_event.is_a?(Hash)
             case last_status_event['Type']
             when 'DONE'
-              # return method (or just don't do anything)
-              return
+              # all went well
+              exception_raised=false
             when 'ERROR'
               Log.log.error("code: #{last_status_event['Code']}")
               if last_status_event['Description']  =~ /bearer token/i
-                Log.log.error("need to regenerate token".red)
+                Log.log.error('need to regenerate token'.red)
                 if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
                   # regenerate token here, expired, or error on it
+                  # Note: in multi-session, each session will have a different one.
                   env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
                 end
               end
               raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
-            else
+            else # case
               raise "unexpected last event type: #{last_status_event['Type']}"
             end
+          else
+            exception_raised=false
+            Log.log.debug('no status read from ascp mgt port')
           end
         rescue SystemCallError => e
           # Process.spawn
@@ -269,7 +296,13 @@ module Aspera
             ascp_pid=nil
             session.delete(:io)
             if !status.success?
-              raise Fasp::Error.new("ascp failed with code #{status.exitstatus}")
+              message="ascp failed with code #{status.exitstatus}"
+              if exception_raised
+                # just debug, as main exception is already here
+                Log.log.debug(message)
+              else
+                raise Fasp::Error.new(message)
+              end
             end
           end
         end # begin-ensure
@@ -283,9 +316,9 @@ module Aspera
       # {'type'=>'DONE'}
       def send_command(job_id,session_index,data)
         job=@jobs[job_id]
-        raise "no such job" if job.nil?
+        raise 'no such job' if job.nil?
         session=job[:sessions][session_index]
-        raise "no such session" if session.nil?
+        raise 'no such session' if session.nil?
         Log.log.debug("command: #{data}")
         # build command
         command=data.
@@ -299,8 +332,8 @@ module Aspera
 
       private
 
-      def initialize(agent_options=nil)
-        agent_options||={}
+      # @param options : keys(symbol): see DEFAULT_OPTIONS
+      def initialize(options=nil)
         super()
         # by default no interactive progress bar
         @quiet=true
@@ -308,9 +341,20 @@ module Aspera
         @jobs={}
         # mutex protects global data accessed by threads
         @mutex=Mutex.new
-        @enable_wss = agent_options[:wss] || false
-        agent_options.delete(:wss)
-        @resume_policy=ResumePolicy.new(agent_options)
+        # set default options and override if specified
+        @options=DEFAULT_OPTIONS.clone
+        if !options.nil?
+          raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
+          options.each do |k,v|
+            if DEFAULT_OPTIONS.has_key?(k)
+              @options[k]=v
+            else
+              raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
+            end
+          end
+        end
+        Log.log.debug("local options= #{options}")
+        @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
       end
 
       # transfer thread entry
@@ -318,7 +362,7 @@ module Aspera
       def transfer_thread_entry(session)
         begin
           # set name for logging
-          Thread.current[:name]="transfer"
+          Thread.current[:name]='transfer'
           Log.log.debug("ENTER (#{Thread.current[:name]})")
           # start transfer with selected resumer policy
           session[:options][:resumer].process do

data/lib/aspera/fasp/manager.rb

@@ -72,6 +72,9 @@ module Aspera
       # start_transfer(transfer_spec,options) : start and wait for completion
       # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
       # optional: shutdown
+      
+      # This checks the validity of the value returned by wait_for_transfers_completion
+      # it must be a list of :success or exception
       def self.validate_status_list(statuses)
         raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
         raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?

data/lib/aspera/fasp/node.rb

@@ -7,9 +7,12 @@ module Aspera
     # this singleton class is used by the CLI to provide a common interface to start a transfer
     # before using it, the use must set the `node_api` member.
     class Node < Manager
-      def initialize(node_api)
+      # option include: root_id if the node is an access key
+      attr_writer :options
+      def initialize(node_api,options={})
         super()
         @node_api=node_api
+        @options=options
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
         @transfer_id=nil
       end
@@ -32,6 +35,25 @@ module Aspera
 
       # generic method
       def start_transfer(transfer_spec,options=nil)
+        # add root id if access key
+        if @options.has_key?(:root_id)
+          case transfer_spec['direction']
+          when 'send';transfer_spec['source_root_id']=@options[:root_id]
+          when 'receive';transfer_spec['destination_root_id']=@options[:root_id]
+          else raise "unexpected direction in ts: #{transfer_spec['direction']}"
+          end
+        end
+        # manage special additional parameter
+        if transfer_spec.has_key?('EX_ssh_key_paths') and transfer_spec['EX_ssh_key_paths'].is_a?(Array) and !transfer_spec['EX_ssh_key_paths'].empty?
+          # not standard, so place standard field
+          if transfer_spec.has_key?('ssh_private_key')
+            Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
+          else
+            Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
+            transfer_spec['ssh_private_key']=File.read(transfer_spec['EX_ssh_key_paths'].first)
+            transfer_spec.delete('EX_ssh_key_paths')
+          end
+        end
         if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
           transfer_spec['tags']['aspera']['xfer_retry']||=150
         end

data/lib/aspera/fasp/parameters.rb

@@ -4,6 +4,7 @@ require 'aspera/temp_file_manager'
 require 'securerandom'
 require 'base64'
 require 'json'
+require 'yaml'
 require 'securerandom'
 require 'fileutils'
 
@@ -12,96 +13,67 @@ module Aspera
     # translate transfer specification to ascp parameter list
     class Parameters
       private
-      # temp folder for file lists, must contain only file lists
+      # Temp folder for file lists, must contain only file lists
       # because of garbage collection takes any file there
       # this could be refined, as , for instance, on macos, temp folder is already user specific
       @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-      PARAM_DEFINITION={
-        # parameters with env vars
-        'remote_password'         => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
-        'token'                   => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
-        'cookie'                  => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
-        'ssh_private_key'         => { :type => :envvar, :variable=>'ASPERA_SCP_KEY'},
-        'EX_at_rest_password'     => { :type => :envvar, :variable=>'ASPERA_SCP_FILEPASS'},
-        'EX_proxy_password'       => { :type => :envvar, :variable=>'ASPERA_PROXY_PASS'},
-        'EX_license_text'         => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
-        # bool params
-        'create_dir'              => { :type => :opt_without_arg, :option_switch=>'-d'},
-        'precalculate_job_size'   => { :type => :opt_without_arg},
-        'keepalive'               => { :type => :opt_without_arg},
-        'delete_before_transfer'  => { :type => :opt_without_arg}, #TODO: doc readme
-        'preserve_access_time'    => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_creation_time'  => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_times'          => { :type => :opt_without_arg}, #TODO: doc
-        'preserve_modification_time'=> { :type => :opt_without_arg}, #TODO: doc
-        'remove_empty_directories'=> { :type => :opt_without_arg}, #TODO: doc
-        'remove_after_transfer'   => { :type => :opt_without_arg}, #TODO: doc
-        'remove_empty_source_directory'=> { :type => :opt_without_arg}, #TODO: doc
-        # value params
-        'cipher'                  => { :type => :opt_with_arg, :option_switch=>'-c',:accepted_types=>String,:encode=>lambda{|cipher|cipher.tr('-','')}},
-        'resume_policy'           => { :type => :opt_with_arg, :option_switch=>'-k',:accepted_types=>String,:default=>'sparse_csum',:translate_values=>{'none'=>0,'attrs'=>1,'sparse_csum'=>2,'full_csum'=>3}},
-        'direction'               => { :type => :opt_with_arg, :option_switch=>'--mode',:accepted_types=>String,:translate_values=>{'receive'=>'recv','send'=>'send'}},
-        'remote_user'             => { :type => :opt_with_arg, :option_switch=>'--user',:accepted_types=>String},
-        'remote_host'             => { :type => :opt_with_arg, :option_switch=>'--host',:accepted_types=>String},
-        'ssh_port'                => { :type => :opt_with_arg, :option_switch=>'-P',:accepted_types=>Integer},
-        'fasp_port'               => { :type => :opt_with_arg, :option_switch=>'-O',:accepted_types=>Integer},
-        'dgram_size'              => { :type => :opt_with_arg, :option_switch=>'-Z',:accepted_types=>Integer},
-        'target_rate_kbps'        => { :type => :opt_with_arg, :option_switch=>'-l',:accepted_types=>Integer},
-        'min_rate_kbps'           => { :type => :opt_with_arg, :option_switch=>'-m',:accepted_types=>Integer},
-        'rate_policy'             => { :type => :opt_with_arg, :option_switch=>'--policy',:accepted_types=>String},
-        'http_fallback'           => { :type => :opt_with_arg, :option_switch=>'-y',:accepted_types=>[String,*Aspera::CommandLineBuilder::BOOLEAN_CLASSES],:translate_values=>{'force'=>'F',true=>1,false=>0}},
-        'http_fallback_port'      => { :type => :opt_with_arg, :option_switch=>'-t',:accepted_types=>Integer},
-        'source_root'             => { :type => :opt_with_arg, :option_switch=>'--source-prefix64',:accepted_types=>String,:encode=>lambda{|prefix|Base64.strict_encode64(prefix)}},
-        'sshfp'                   => { :type => :opt_with_arg, :option_switch=>'--check-sshfp',:accepted_types=>String},
-        'symlink_policy'          => { :type => :opt_with_arg, :option_switch=>'--symbolic-links',:accepted_types=>String},
-        'overwrite'               => { :type => :opt_with_arg, :accepted_types=>String},
-        'exclude_newer_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
-        'exclude_older_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
-        'preserve_acls'           => { :type => :opt_with_arg, :accepted_types=>String},
-        'move_after_transfer'     => { :type => :opt_with_arg, :accepted_types=>String},
-        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
-        # non standard parameters
-        'EX_fasp_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
-        'EX_http_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
-        'EX_ssh_key_paths'        => { :type => :opt_with_arg, :option_switch=>'-i',:accepted_types=>Array},
-        'EX_http_transfer_jpeg'   => { :type => :opt_with_arg, :option_switch=>'-j',:accepted_types=>Integer},
-        'EX_multi_session_part'   => { :type => :opt_with_arg, :option_switch=>'-C',:accepted_types=>String},
-        'EX_no_read'              => { :type => :opt_without_arg, :option_switch=>'--no-read'},
-        'EX_no_write'             => { :type => :opt_without_arg, :option_switch=>'--no-write'},
-        'EX_apply_local_docroot'  => { :type => :opt_without_arg, :option_switch=>'--apply-local-docroot'},
-        # TODO: manage those parameters, some are for connect only ? node api ?
-        'target_rate_cap_kbps'    => { :type => :ignore, :accepted_types=>Integer},
-        'target_rate_percentage'  => { :type => :ignore, :accepted_types=>String}, # -wf -l<rate>p
-        'min_rate_cap_kbps'       => { :type => :ignore, :accepted_types=>Integer},
-        'rate_policy_allowed'     => { :type => :ignore, :accepted_types=>String},
-        'fasp_url'                => { :type => :ignore, :accepted_types=>String},
-        'lock_rate_policy'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'lock_min_rate'           => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'lock_target_rate'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        #'authentication'          => { :type => :ignore, :accepted_types=>String}, # = token
-        'https_fallback_port'     => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
-        'content_protection'      => { :type => :ignore, :accepted_types=>String},
-        'cipher_allowed'          => { :type => :ignore, :accepted_types=>String},
-        'multi_session'           => { :type => :ignore, :accepted_types=>Integer}, # managed
-        # optional tags (  additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'}  )
-        'tags'                    => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
-        # special processing @builder.process_param( called individually
-        'use_ascp4'               => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'paths'                   => { :type => :defer, :accepted_types=>Array},
-        'EX_file_list'            => { :type => :defer, :option_switch=>'--file-list', :accepted_types=>String},
-        'EX_file_pair_list'       => { :type => :defer, :option_switch=>'--file-pair-list', :accepted_types=>String},
-        'EX_ascp_args'            => { :type => :defer, :accepted_types=>Array},
-        'destination_root'        => { :type => :defer, :accepted_types=>String},
-        'wss_enabled'             => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        'wss_port'                => { :type => :defer, :accepted_types=>Integer},
-      }
-
-      private_constant :PARAM_DEFINITION
+      @@param_description_cache=nil
+      # @return normaiwed description of transfer spec parameters
+      def self.description
+        return @@param_description_cache unless @@param_description_cache.nil?
+        # config file in same folder with same name as this source
+        @@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
+        Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
+      end
+
+      # Agents shown in manual
+      SUPPORTED_AGENTS=[:direct,:node,:connect]
+      # Short names of columns in manual
+      SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
+
+      # @return a table suitable to display a manual
+      def self.man_table
+        result=[]
+        description.keys.map do |k|
+          i=description[k]
+          param={name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
+          SUPPORTED_AGENTS.each do |a|
+            param[a.to_s[0].to_sym]=i[:context].nil? || i[:context].include?(a) ? 'Y' : ''
+          end
+          # only keep lines that are usable in supported agents
+          next if SUPPORTED_AGENTS_SHORT.inject(true){|m,i|m and param[i].empty?}
+          param[:cli]=case i[:cltype]
+          when :envvar; 'env:'+i[:clvarname]
+          when :opt_without_arg,:opt_with_arg; i[:option_switch]
+          else ''
+          end
+          if i.has_key?(:enum)
+            param[:description] << "\nAllowed values: #{i[:enum].join(', ')}"
+          end
+          result.push(param)
+        end
+        return result
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_cipher(v)
+        v.tr('-','')
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_source_root(v)
+        Base64.strict_encode64(v)
+      end
+
+      # special encoding methods used in YAML (key: :encode)
+      def self.encode_tags(v)
+        Base64.strict_encode64(JSON.generate(v))
+      end
 
       def initialize(job_spec,options)
         @job_spec=job_spec
-        @builder=Aspera::CommandLineBuilder.new(@job_spec,PARAM_DEFINITION)
         @options=options
+        @builder=Aspera::CommandLineBuilder.new(@job_spec,self.class.description)
       end
 
       public
@@ -124,7 +96,7 @@ module Aspera
         # special cases
         @job_spec.delete('source_root') if @job_spec.has_key?('source_root') and @job_spec['source_root'].empty?
 
-        # use web socket initiation ?
+        # use web socket session initiation ?
         if @builder.process_param('wss_enabled',:get_value) and @options[:wss]
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
@@ -150,8 +122,7 @@ module Aspera
           # destination will be base64 encoded, put before path arguments
           @builder.add_command_line_options(['--dest64'])
         end
-
-        PARAM_DEFINITION['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
+        @builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
         paths_array=@builder.process_param('paths',:get_value)
         unless paths_array.nil?
           # use file list if there is storage defined for it.