aspera-cli 4.2.0 → 4.4.0

data/lib/aspera/cli/plugins/preview.rb

@@ -6,6 +6,8 @@ require 'aspera/preview/file_types'
 require 'aspera/persistency_action_once'
 require 'aspera/node'
 require 'aspera/hash_ext'
+require 'aspera/timer_limiter'
+require 'aspera/id_generator'
 require 'date'
 require 'securerandom'
 
@@ -24,7 +26,8 @@ module Aspera
         DEFAULT_PREVIEWS_FOLDER='previews'
         AK_MARKER_FILE='.aspera_access_key'
         LOCAL_STORAGE_PCVL='file:///'
-        private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE
+        LOG_LIMITER_SEC=30.0
+        private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
 
         # option_skip_format has special accessors
         attr_accessor :option_previews_folder
@@ -40,6 +43,8 @@ module Aspera
           @preview_formats_to_generate=Aspera::Preview::Generator::PREVIEW_FORMATS.clone
           # options for generation
           @gen_options=Aspera::Preview::Options.new
+          # used to trigger periodic processing
+          @periodic=TimerLimiter.new(LOG_LIMITER_SEC)
           # link CLI options to gen_info attributes
           self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
           self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
@@ -113,13 +118,14 @@ module Aspera
         end
 
         # old version based on folders
-        def process_trevents(iteration_token)
+        # @param iteration_persistency can be nil
+        def process_trevents(iteration_persistency)
           events_filter={
             'access_key'=>@access_key_self['id'],
             'type'=>'download.ended'
           }
-          # optionally by iteration token
-          events_filter['iteration_token']=iteration_token unless iteration_token.nil?
+          # optionally add iteration token from persistency
+          events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
           begin
             events=@api_node.read('events',events_filter)[:data]
           rescue RestCallError => e
@@ -132,47 +138,63 @@ module Aspera
           end
           return if events.empty?
           events.each do |event|
-            next unless event['data']['direction'].eql?('receive')
-            next unless event['data']['status'].eql?('completed')
-            next unless event['data']['error_code'].eql?(0)
-            next unless event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
-            folder_id=event.dig('data','tags','aspera','node','file_id')
-            folder_id||=event.dig('data','file_id')
-            next if folder_id.nil?
-            folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
-            next if folder_entry.nil?
-            scan_folder_files(folder_entry)
+            if event['data']['direction'].eql?('receive') and
+            event['data']['status'].eql?('completed') and
+            event['data']['error_code'].eql?(0) and
+            event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
+              folder_id=event.dig('data','tags','aspera','node','file_id')
+              folder_id||=event.dig('data','file_id')
+              if !folder_id.nil?
+                folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
+                scan_folder_files(folder_entry) unless folder_entry.nil?
+              end
+            end
+            if @periodic.trigger? or event.equal?(events.last)
+              Log.log.info("Processed event #{event['id']}")
+              # save checkpoint to avoid losing processing in case of error
+              if !iteration_persistency.nil?
+                iteration_persistency.data[0]=event['id'].to_s
+                iteration_persistency.save
+              end
+            end
           end
-          return events.last['id'].to_s
         end
 
         # requests recent events on node api and process newly modified folders
-        def process_events(iteration_token)
+        def process_events(iteration_persistency)
           # get new file creation by access key (TODO: what if file already existed?)
           events_filter={
             'access_key'=>@access_key_self['id'],
             'type'=>'file.*'
           }
-          # and optionally by iteration token
-          events_filter['iteration_token']=iteration_token unless iteration_token.nil?
+          # optionally add iteration token from persistency
+          events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
           events=@api_node.read('events',events_filter)[:data]
           return if events.empty?
           events.each do |event|
             # process only files
-            next unless event.dig('data','type').eql?('file')
-            file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
-            next if file_entry.nil?
-            next unless @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
-            file_entry['parent_file_id']=event['data']['parent_file_id']
-            if event['types'].include?('file.deleted')
-              Log.log.error('TODO'.red)
+            if event.dig('data','type').eql?('file')
+              file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
+              if !file_entry.nil? and
+              @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
+                file_entry['parent_file_id']=event['data']['parent_file_id']
+                if event['types'].include?('file.deleted')
+                  Log.log.error('TODO'.red)
+                end
+                if event['types'].include?('file.deleted')
+                  generate_preview(file_entry)
+                end
+              end
             end
-            if event['types'].include?('file.deleted')
-              generate_preview(file_entry)
+            if @periodic.trigger? or event.equal?(events.last)
+              Log.log.info("Processing event #{event['id']}")
+              # save checkpoint to avoid losing processing in case of error
+              if !iteration_persistency.nil?
+                iteration_persistency.data[0]=event['id'].to_s
+                iteration_persistency.save
+              end
             end
           end
-          # write new iteration file
-          return events.last['id'].to_s
         end
 
         def do_transfer(direction,folder_id,source_filename,destination='/')
@@ -180,15 +202,17 @@ module Aspera
           if @default_transfer_spec.nil?
             # make a dummy call to get some default transfer parameters
             res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
-            sample_transfer_spec=res[:data]['transfer_specs'].first['transfer_spec']
+            template_ts=res[:data]['transfer_specs'].first['transfer_spec']
             # get ports, anyway that should be 33001 for both. add remote_user ?
-            @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=sample_transfer_spec[e];h}
+            @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=template_ts[e];h}
+            if ! @default_transfer_spec['remote_user'].eql?(Aspera::Node::ACCESS_KEY_TRANSFER_USER)
+              Log.log.warn("remote_user shall be xfer")
+              @default_transfer_spec['remote_user']=Aspera::Node::ACCESS_KEY_TRANSFER_USER
+            end
+            Aspera::Node::set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],self.options.get_option(:password,:mandatory))
             # note: we use the same address for ascp than for node api instead of the one from upload_setup
-            @default_transfer_spec.merge!({
-              'token'            => "Basic #{Base64.strict_encode64("#{@access_key_self['id']}:#{self.options.get_option(:password,:mandatory)}")}",
-              'remote_host'      => @transfer_server_address,
-              'remote_user'      => Fasp::ACCESS_KEY_TRANSFER_USER
-            })
+            # TODO: configurable ? useful ?
+            @default_transfer_spec['remote_host']=@transfer_server_address
           end
           tspec=@default_transfer_spec.merge({
             'direction'  => direction,
@@ -338,40 +362,45 @@ module Aspera
             entry=entries_to_process.shift
             # process this entry only if it is within the scan_start
             entry_path_with_slash=entry['path']
+            Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
             entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
             if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
               Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
               next
             end
             Log.log.debug("item:#{entry}")
-            case entry['type']
-            when 'file'
-              if filter_block.call(entry)
-                generate_preview(entry)
-              else
-                Log.log.debug('skip by filter')
-              end
-            when 'link'
-              Log.log.debug('Ignoring link.')
-            when 'folder'
-              if @option_skip_folders.include?(entry['path'])
-                Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
-              else
-                Log.log.debug("#{entry['path']} folder".green)
-                # get folder content
-                folder_entries=get_folder_entries(entry['id'])
-                # process all items in current folder
-                folder_entries.each do |folder_entry|
-                  # add path for older versions of ES
-                  if !folder_entry.has_key?('path')
-                    folder_entry['path']=entry_path_with_slash+folder_entry['name']
+            begin
+              case entry['type']
+              when 'file'
+                if filter_block.call(entry)
+                  generate_preview(entry)
+                else
+                  Log.log.debug('skip by filter')
+                end
+              when 'link'
+                Log.log.debug('Ignoring link.')
+              when 'folder'
+                if @option_skip_folders.include?(entry['path'])
+                  Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
+                else
+                  Log.log.debug("#{entry['path']} folder".green)
+                  # get folder content
+                  folder_entries=get_folder_entries(entry['id'])
+                  # process all items in current folder
+                  folder_entries.each do |folder_entry|
+                    # add path for older versions of ES
+                    if !folder_entry.has_key?('path')
+                      folder_entry['path']=entry_path_with_slash+folder_entry['name']
+                    end
+                    folder_entry['parent_file_id']=entry['id']
+                    entries_to_process.push(folder_entry)
                   end
-                  folder_entry['parent_file_id']=entry['id']
-                  entries_to_process.push(folder_entry)
                 end
+              else
+                Log.log.warn("unknown entry type: #{entry['type']}")
               end
-            else
-              Log.log.warn("unknown entry type: #{entry['type']}")
+            rescue => e
+              Log.log.warn("An error occured: #{e}, ignoring")
             end
           end
         end
@@ -437,18 +466,15 @@ module Aspera
             scan_folder_files(folder_info,scan_path)
             return Main.result_status('scan finished')
           when :events,:trevents
-            iteration_data=[]
             iteration_persistency=nil
             if self.options.get_option(:once_only,:mandatory)
               iteration_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
-              data: iteration_data,
-              ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
+              data:    [],
+              id:      IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
             end
-
-            # call method specified
-            iteration_data[0]=send("process_#{command}",iteration_data[0])
-            iteration_persistency.save unless iteration_persistency.nil?
+            # call processing method specified by command line command
+            send("process_#{command}",iteration_persistency)
             return Main.result_status("#{command} finished")
           when :check
             Aspera::Preview::Utils.check_tools(@skip_types)

data/lib/aspera/cli/plugins/server.rb

@@ -1,5 +1,6 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/ascmd'
+require 'aspera/node'
 require 'aspera/ssh'
 require 'aspera/nagios'
 require 'tempfile'
@@ -67,13 +68,15 @@ module Aspera
         def execute_action
           server_uri=URI.parse(self.options.get_option(:url,:mandatory))
           Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
+          server_transfer_spec={'remote_host'=>server_uri.hostname}
           shell_executor=nil
           case server_uri.scheme
           when 'ssh'
-            server_transfer_spec={
-              'remote_host'=>server_uri.hostname,
-              'remote_user'=>self.options.get_option(:username,:mandatory),
-            }
+            if self.options.get_option(:username,:optional).nil?
+              self.options.set_option(:username,Aspera::Node::ACCESS_KEY_TRANSFER_USER)
+              Log.log.info("Using default transfer user: #{Aspera::Node::ACCESS_KEY_TRANSFER_USER}")
+            end
+            server_transfer_spec['remote_user']=self.options.get_option(:username,:mandatory)
             ssh_options=self.options.get_option(:ssh_options,:optional)
             raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
             if !server_uri.port.nil?
@@ -102,8 +105,16 @@ module Aspera
                 cred_set=true
               end
             end
-            raise 'either password or key must be provided' if !cred_set
+            # if user provided transfer spec has a token, we will use by pass keys
+            cred_set=true if self.transfer.option_transfer_spec['token'].is_a?(String)
+            raise 'either password, key , or transfer spec token must be provided' if !cred_set
             shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
+          when 'https'
+            raise "ERROR: transfer spec with token required" unless self.transfer.option_transfer_spec['token'].is_a?(String)
+            server_transfer_spec.merge!({
+              'wss_enabled'=>true,
+              'wss_port'   =>server_uri.port
+            })
           when 'local'
             shell_executor=LocalExecutor.new
           else

data/lib/aspera/cli/transfer_agent.rb

@@ -1,7 +1,7 @@
 require 'aspera/fasp/local'
+require 'aspera/fasp/parameters'
 require 'aspera/fasp/connect'
 require 'aspera/fasp/node'
-require 'aspera/fasp/aoc'
 require 'aspera/fasp/http_gw'
 require 'aspera/cli/listener/logger'
 require 'aspera/cli/listener/progress_multi'
@@ -16,33 +16,47 @@ module Aspera
       FILE_LIST_FROM_ARGS='@args'
       # special value for --sources : read file list from transfer spec (--ts)
       FILE_LIST_FROM_TRANSFER_SPEC='@ts'
-      private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC
-      # @param cli_objects external objects: option manager, config file manager
-      def initialize(cli_objects)
-        @opt_mgr=cli_objects[:options]
-        @config=cli_objects[:config]
-        # transfer spec overrides provided on command line
-        @transfer_spec_cmdline={"create_dir"=>true}
+      DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
+From: <%=from_name%> <<%=from_email%>>
+To: <<%=to%>>
+Subject: <%=subject%>
+
+Transfer is: <%=global_transfer_status%>
+
+<%=ts.to_yaml%>
+END_OF_TEMPLATE
+      #%
+      private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
+      # @param env external objects: option manager, config file manager
+      def initialize(env)
+        # same as plugin environment
+        @env=env
+        # command line can override transfer spec
+        @transfer_spec_cmdline={'create_dir'=>true}
         # the currently selected transfer agent
         @agent=nil
         @progress_listener=Listener::ProgressMulti.new
         # source/destination pair, like "paths" of transfer spec
         @transfer_paths=nil
-        @opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
-        @opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
-        @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
-        @opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
-        @opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
-        @opt_mgr.add_opt_simple(:transfer_info,"additional information for transfer client")
-        @opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
-        @opt_mgr.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
-        @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
-        @opt_mgr.set_option(:transfer,:direct)
-        @opt_mgr.set_option(:src_type,:list)
-        @opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
-        @opt_mgr.parse_options!
+        options.set_obj_attr(:ts,self,:option_transfer_spec)
+        options.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{options.get_option(:ts,:optional)}")
+        options.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{options.get_option(:local_resume,:optional)}")
+        options.add_opt_simple(:to_folder,"destination folder for downloaded files")
+        options.add_opt_simple(:sources,"list of source files (see doc)")
+        options.add_opt_simple(:transfer_info,"parameters for transfer agent")
+        options.add_opt_list(:src_type,[:list,:pair],"type of file list")
+        options.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node],"type of transfer agent")
+        options.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
+        options.set_option(:transfer,:direct)
+        options.set_option(:src_type,:list)
+        options.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
+        options.parse_options!
       end
 
+      def options; @env[:options];end
+
+      def config; @env[:config];end
+
       def option_transfer_spec; @transfer_spec_cmdline; end
 
       # multiple option are merged
@@ -54,8 +68,8 @@ module Aspera
         @agent=instance
         @agent.add_listener(Listener::Logger.new)
         # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
-        if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
-        (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and !@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct))
+        if options.get_option(:progress,:mandatory).eql?(:multi) or
+        (options.get_option(:progress,:mandatory).eql?(:native) and !options.get_option(:transfer,:mandatory).eql?(:direct))
           @agent.add_listener(@progress_listener)
         end
       end
@@ -63,15 +77,15 @@ module Aspera
       # analyze options and create new agent if not already created or set
       def set_agent_by_options
         return nil unless @agent.nil?
-        agent_type=@opt_mgr.get_option(:transfer,:mandatory)
+        agent_type=options.get_option(:transfer,:mandatory)
         case agent_type
         when :direct
-          agent_options=@opt_mgr.get_option(:transfer_info,:optional)
+          agent_options=options.get_option(:transfer_info,:optional)
           agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
           new_agent=Fasp::Local.new(agent_options)
-          new_agent.quiet=false if @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
+          new_agent.quiet=false if options.get_option(:progress,:mandatory).eql?(:native)
         when :httpgw
-          httpgw_config=@opt_mgr.get_option(:transfer_info,:mandatory)
+          httpgw_config=options.get_option(:transfer_info,:mandatory)
           new_agent=Fasp::HttpGW.new(httpgw_config)
         when :connect
           new_agent=Fasp::Connect.new
@@ -79,45 +93,37 @@ module Aspera
           # way for code to setup alternate node api in advance
           # support: @preset:<name>
           # support extended values
-          node_config=@opt_mgr.get_option(:transfer_info,:optional)
+          node_config=options.get_option(:transfer_info,:optional)
           # if not specified: use default node
           if node_config.nil?
-            param_set_name=@config.get_plugin_default_config_name(:node)
+            param_set_name=config.get_plugin_default_config_name(:node)
             raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
-            node_config=@config.preset_by_name(param_set_name)
+            node_config=config.preset_by_name(param_set_name)
           end
           Log.log.debug("node=#{node_config}")
-          raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
-          # now check there are required parameters
-          sym_config=[:url,:username,:password].inject({}) do |h,param|
-            raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
-            h[param]=node_config[param.to_s]
-            h
+          raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" unless node_config.is_a?(Hash)
+          # here, node_config is a Hash
+          node_config=node_config.symbolize_keys
+          # Check mandatory params
+          [:url,:username,:password].each { |k| raise CliBadArgument,"missing parameter [#{k}] in node specification: #{node_config}" unless node_config.has_key?(k) }
+          if node_config[:password].match(/^Bearer /)
+            node_api=Rest.new({
+              base_url: node_config[:url],
+              headers: {
+              'X-Aspera-AccessKey'=>node_config[:username],
+              'Authorization'     =>node_config[:password]}})
+          else
+            node_api=Rest.new({
+              base_url: node_config[:url],
+              auth:     {
+              type:     :basic,
+              username: node_config[:username],
+              password: node_config[:password]
+              }})
           end
-          node_api=Rest.new({
-            :base_url => sym_config[:url],
-            :auth     => {
-            :type     =>:basic,
-            :username => sym_config[:username],
-            :password => sym_config[:password]
-            }})
           new_agent=Fasp::Node.new(node_api)
-        when :aoc
-          aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
-          if aoc_config.nil?
-            param_set_name=@config.get_plugin_default_config_name(:aspera)
-            raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
-            aoc_config=@config.preset_by_name(param_set_name)
-          end
-          Log.log.debug("aoc=#{aoc_config}")
-          raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
-          # convert keys from string (config) to symbol (agent)
-          aoc_config=aoc_config.symbolize_keys
-          # convert auth value from string (config) to symbol (agent)
-          aoc_config[:auth]=aoc_config[:auth].to_sym if aoc_config[:auth].is_a?(String)
-          # private key could be @file:... in config
-          aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
-          new_agent=Fasp::Aoc.new(aoc_config)
+          # add root id if it's an access key
+          new_agent.options={root_id: node_config[:root_id]} if node_config.has_key?(:root_id)
         else
           raise "Unexpected transfer agent type: #{agent_type}"
         end
@@ -129,7 +135,7 @@ module Aspera
       # sets default if needed
       # param: 'send' or 'receive'
       def destination_folder(direction)
-        dest_folder=@opt_mgr.get_option(:to_folder,:optional)
+        dest_folder=options.get_option(:to_folder,:optional)
         return dest_folder unless dest_folder.nil?
         dest_folder=@transfer_spec_cmdline['destination_root']
         return dest_folder unless dest_folder.nil?
@@ -152,16 +158,17 @@ module Aspera
         # start with lower priority : get paths from transfer spec on command line
         @transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
         # is there a source list option ?
-        file_list=@opt_mgr.get_option(:sources,:optional)
+        file_list=options.get_option(:sources,:optional)
         case file_list
         when nil,FILE_LIST_FROM_ARGS
           Log.log.debug("getting file list as parameters")
           # get remaining arguments
-          file_list=@opt_mgr.get_next_argument("source file list",:multiple)
+          file_list=options.get_next_argument("source file list",:multiple)
           raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
         when FILE_LIST_FROM_TRANSFER_SPEC
           Log.log.debug("assume list provided in transfer spec")
-          raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil?
+          special_case_direct_with_list=options.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
+          raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
           # here we assume check of sources is made in transfer agent
           return @transfer_paths
         when Array
@@ -174,7 +181,7 @@ module Aspera
         if !@transfer_paths.nil?
           Log.log.warn("--sources overrides paths from --ts")
         end
-        case @opt_mgr.get_option(:src_type,:mandatory)
+        case options.get_option(:src_type,:mandatory)
         when :list
           # when providing a list, just specify source
           @transfer_paths=file_list.map{|i|{'source'=>i}}
@@ -189,20 +196,20 @@ module Aspera
 
       # start a transfer and wait for completion, plugins shall use this method
       # @param transfer_spec
-      # @param options specific options for the transfer_agent
-      # options[:src] specifies how destination_root is set (how transfer spec was generated)
+      # @param tr_opts specific options for the transfer_agent
+      # tr_opts[:src] specifies how destination_root is set (how transfer spec was generated)
       # other options are carried to specific agent
-      def start(transfer_spec,options)
+      def start(transfer_spec,tr_opts)
         # check parameters
         raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
-        raise "options must be hash" unless options.is_a?(Hash)
+        raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
         # process :src option
         case transfer_spec['direction']
         when 'receive'
           # init default if required in any case
           @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
         when 'send'
-          case options[:src]
+          case tr_opts[:src]
           when :direct
             # init default if required
             @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
@@ -213,12 +220,12 @@ module Aspera
           when :node_gen4
             @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
           else
-            raise StandardError,"InternalError: unsupported value: #{options[:src]}"
+            raise StandardError,"InternalError: unsupported value: #{tr_opts[:src]}"
           end
         end
 
         # only used here
-        options.delete(:src)
+        tr_opts.delete(:src)
 
         # update command line paths, unless destination already has one
         @transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
@@ -227,13 +234,26 @@ module Aspera
         # create transfer agent
         self.set_agent_by_options
         Log.log.debug("transfer agent is a #{@agent.class}")
-        @agent.start_transfer(transfer_spec,options)
+        @agent.start_transfer(transfer_spec,tr_opts)
         result=@agent.wait_for_transfers_completion
         @progress_listener.reset
         Fasp::Manager.validate_status_list(result)
+        send_email_transfer_notification(transfer_spec,result)
         return result
       end
 
+      def send_email_transfer_notification(transfer_spec,statuses)
+        return if options.get_option(:notif_to,:optional).nil?
+        global_status=self.class.session_status(statuses)
+        email_vars={
+          global_transfer_status: global_status,
+          subject: "ascli transfer: #{global_status}",
+          body: "Transfer is: #{global_status}",
+          ts: transfer_spec
+        }
+        @env[:config].send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
+      end
+
       # @return :success if all sessions statuses returned by "start" are success
       # else return the first error exception object
       def self.session_status(statuses)

data/lib/aspera/cli/version.rb

@@ -1,5 +1,5 @@
 module Aspera
   module Cli
-    VERSION = "4.2.0"
+    VERSION = "4.4.0"
   end
 end