aspera-cli 4.0.0.pre1

data/lib/aspera/cli/plugins/xnode.rb

@@ -0,0 +1,115 @@
+require 'aspera/cli/basic_auth_plugin'
+require 'aspera/persistency_action_once'
+require "base64"
+
+module Aspera
+  module Cli
+    module Plugins
+      # experiments
+      class Xnode < BasicAuthPlugin
+        def initialize(env)
+          super(env)
+          self.options.add_opt_simple(:filter_transfer,"Ruby expression for filter at transfer level (cleanup)")
+          self.options.add_opt_simple(:filter_file,"Ruby expression for filter at file level (cleanup)")
+          self.options.parse_options!
+        end
+        # "transfer_filter"=>"t['status'].eql?('completed') and t['start_spec']['remote_user'].eql?('faspex')", :file_filter=>"f['status'].eql?('completed') and 0 != f['size'] and t['start_spec']['direction'].eql?('send')"
+
+        ACTIONS=[ :postprocess, :cleanup, :forward ]
+
+        # retrieve tranfer list using API and persistency file
+        def self.get_transfers_iteration(api_node,params)
+          # array with one element max
+          iteration_data=[]
+          iteration_persistency=nil
+          if self.options.get_option(:once_only,:mandatory)
+            iteration_persistency=PersistencyActionOnce.new(
+            manager: @agents[:persistency],
+            data: iteration_data,
+            ids:  ['xnode',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
+          end
+          iteration_data[0]=process_file_events(iteration_data[0])
+          # first time run ? or subsequent run ?
+          params[:iteration_token]=iteration_data[0] unless iteration_data[0].nil?
+          resp=api_node.read('ops/transfers',params)
+          transfers=resp[:data]
+          if transfers.is_a?(Array) then
+            # 3.7.2, released API
+            iteration_data[0]=URI.decode_www_form(URI.parse(resp[:http]['Link'].match(/<([^>]+)>/)[1]).query).to_h['iteration_token']
+          else
+            # 3.5.2, deprecated API
+            iteration_data[0]=transfers['iteration_token']
+            transfers=transfers['transfers']
+          end
+          iteration_persistency.save unless iteration_persistency.nil?
+          return transfers
+        end
+
+        def execute_action
+          api_node=Rest.new({
+            :base_url => self.options.get_option(:url,:mandatory),
+            :auth     => {
+            :type     => :basic,
+            :username => self.options.get_option(:username,:mandatory),
+            :password => self.options.get_option(:password,:mandatory)
+            }})
+          command=self.options.get_next_command(ACTIONS)
+          case command
+          when :cleanup
+            transfers=self.class.get_transfers_iteration(api_node,{:active_only=>false})
+            filter_transfer=self.options.get_option(:filter_transfer,:mandatory)
+            filter_file=self.options.get_option(:filter_file,:mandatory)
+            Log.log.debug("filter_transfer: #{filter_transfer}")
+            Log.log.debug("filter_file: #{filter_file}")
+            # build list of files to delete: non zero files, downloads, for specified user
+            paths_to_delete=[]
+            transfers.each do |t|
+              if eval(filter_transfer)
+                t['files'].each do |f|
+                  if eval(filter_file)
+                    if !paths_to_delete.include?(f['path'])
+                      paths_to_delete.push(f['path'])
+                      Log.log.info("to delete: #{f['path']}")
+                    end
+                  end
+                end
+              end
+            end
+            # delete files, if any
+            if paths_to_delete.length != 0
+              Log.log.info("deletion")
+              return self.delete_files(api_node,paths_to_delete,nil)
+            else
+              Log.log.info("nothing to delete")
+            end
+            return Main.result_nothing
+          when :forward
+            # detect transfer sessions since last call
+            transfers=self.class.get_transfers_iteration(api_node,{:active_only=>false})
+            # build list of all files received in all sessions
+            filelist=[]
+            transfers.select { |t| t['status'].eql?('completed') and t['start_spec']['direction'].eql?('receive') }.each do |t|
+              t['files'].each { |f| filelist.push(f['path']) }
+            end
+            if filelist.empty?
+              Log.log.debug("NO TRANSFER".red)
+              return Main.result_nothing
+            end
+            Log.log.debug("file list=#{filelist}")
+            # get download transfer spec on destination node
+            transfer_params={ :transfer_requests => [ { :transfer_request => { :paths => filelist.map {|i| {:source=>i} } } } ] }
+            send_result=api_node.call({:operation=>'POST',:subpath=>'files/download_setup',:json_params=>transfer_params})
+            # only one request, so only one answer
+            transfer_spec=send_result[:data]['transfer_specs'].first['transfer_spec']
+            # execute transfer
+            return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
+          when :postprocess
+            transfers=self.class.get_transfers_iteration(api_node,{:view=>'summary',:direction=>'receive',:active_only=>false})
+            return { :type=>:object_list,:data => transfers }
+          end # case command
+          raise "ERROR: shall not reach this line"
+        end # execute_action
+      end # Main
+    end # Plugin
+  end # Cli
+end # Aspera

data/lib/aspera/cli/transfer_agent.rb

@@ -0,0 +1,251 @@
+require 'aspera/fasp/local'
+require 'aspera/fasp/connect'
+require 'aspera/fasp/node'
+require 'aspera/fasp/aoc'
+require 'aspera/fasp/http_gw'
+require 'aspera/cli/listener/logger'
+require 'aspera/cli/listener/progress_multi'
+
+module Aspera
+  module Cli
+    # The Transfer agent is a common interface to start a transfer using
+    # one of the supported transfer agents
+    # provides CLI options to select one of the transfer agents (fasp client)
+    class TransferAgent
+      # special value for --sources : read file list from arguments
+      FILE_LIST_FROM_ARGS='@args'
+      # special value for --sources : read file list from transfer spec (--ts)
+      FILE_LIST_FROM_TRANSFER_SPEC='@ts'
+      private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC
+      # @param cli_objects external objects: option manager, config file manager
+      def initialize(cli_objects)
+        @opt_mgr=cli_objects[:options]
+        @config=cli_objects[:config]
+        # transfer spec overrides provided on command line
+        @transfer_spec_cmdline={}
+        # the currently selected transfer agent
+        @agent=nil
+        @progress_listener=Listener::ProgressMulti.new
+        # source/destination pair, like "paths" of transfer spec
+        @transfer_paths=nil
+        @opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
+        @opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
+        @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
+        @opt_mgr.add_opt_simple(:to_folder,"destination folder for downloaded files")
+        @opt_mgr.add_opt_simple(:sources,"list of source files (see doc)")
+        @opt_mgr.add_opt_simple(:transfer_info,"additional information for transfer client")
+        @opt_mgr.add_opt_list(:src_type,[:list,:pair],"type of file list")
+        @opt_mgr.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node,:aoc],"type of transfer")
+        @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
+        @opt_mgr.set_option(:transfer,:direct)
+        @opt_mgr.set_option(:src_type,:list)
+        @opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
+        @opt_mgr.parse_options!
+      end
+
+      def option_transfer_spec; @transfer_spec_cmdline; end
+
+      def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
+
+      def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
+
+      def set_agent_instance(instance)
+        @agent=instance
+        @agent.add_listener(Listener::Logger.new)
+        # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
+        if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) or
+        (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) and !@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct))
+          @agent.add_listener(@progress_listener)
+        end
+      end
+
+      # analyze options and create new agent if not already created or set
+      def set_agent_by_options
+        return nil unless @agent.nil?
+        agent_type=@opt_mgr.get_option(:transfer,:mandatory)
+        case agent_type
+        when :direct
+          agent_options=@opt_mgr.get_option(:transfer_info,:optional)
+          agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
+          new_agent=Fasp::Local.new(agent_options)
+          new_agent.quiet=false if @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
+        when :httpgw
+          httpgw_config=@opt_mgr.get_option(:transfer_info,:mandatory)
+          new_agent=Fasp::HttpGW.new(httpgw_config)
+        when :connect
+          new_agent=Fasp::Connect.new
+        when :node
+          # way for code to setup alternate node api in avance
+          # support: @preset:<name>
+          # support extended values
+          node_config=@opt_mgr.get_option(:transfer_info,:optional)
+          # if not specified: use default node
+          if node_config.nil?
+            param_set_name=@config.get_plugin_default_config_name(:node)
+            raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
+            node_config=@config.preset_by_name(param_set_name)
+          end
+          Log.log.debug("node=#{node_config}")
+          raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
+          # now check there are required parameters
+          sym_config=[:url,:username,:password].inject({}) do |h,param|
+            raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
+            h[param]=node_config[param.to_s]
+            h
+          end
+          node_api=Rest.new({
+            :base_url => sym_config[:url],
+            :auth     => {
+            :type     =>:basic,
+            :username => sym_config[:username],
+            :password => sym_config[:password]
+            }})
+          new_agent=Fasp::Node.new(node_api)
+        when :aoc
+          aoc_config=@opt_mgr.get_option(:transfer_info,:optional)
+          if aoc_config.nil?
+            param_set_name=@config.get_plugin_default_config_name(:aspera)
+            raise CliBadArgument,"No default AoC configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
+            aoc_config=@config.preset_by_name(param_set_name)
+          end
+          Log.log.debug("aoc=#{aoc_config}")
+          raise CliBadArgument,"the aoc configuration shall be Hash, not #{aoc_config.class} (#{aoc_config}), refer to manual" if !aoc_config.is_a?(Hash)
+          # convert keys from string (config) to symbol (agent)
+          aoc_config=aoc_config.symbolize_keys
+          # convert auth value from string (config) to symbol (agent)
+          aoc_config[:auth]=aoc_config[:auth].to_sym if aoc_config[:auth].is_a?(String)
+          # private key could be @file:... in config
+          aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
+          new_agent=Fasp::Aoc.new(aoc_config)
+        else
+          raise "INTERNAL ERROR"
+        end
+        set_agent_instance(new_agent)
+        return nil
+      end
+
+      # return destination folder for transfers
+      # sets default if needed
+      # param: 'send' or 'receive'
+      def destination_folder(direction)
+        dest_folder=@opt_mgr.get_option(:to_folder,:optional)
+        return dest_folder unless dest_folder.nil?
+        dest_folder=@transfer_spec_cmdline['destination_root']
+        return dest_folder unless dest_folder.nil?
+        # default: / on remote, . on local
+        case direction.to_s
+        when 'send';dest_folder='/'
+        when 'receive';dest_folder='.'
+        else raise "wrong direction: #{direction}"
+        end
+        return dest_folder
+      end
+
+      # This is how the list of files to be transfered is specified
+      # get paths suitable for transfer spec from command line
+      # @return {:source=>(mandatory), :destination=>(optional)}
+      # computation is done only once, cache is kept in @transfer_paths
+      def ts_source_paths
+        # return cache if set
+        return @transfer_paths unless @transfer_paths.nil?
+        # start with lower priority : get paths from transfer spec on command line
+        @transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
+        # is there a source list option ?
+        file_list=@opt_mgr.get_option(:sources,:optional)
+        case file_list
+        when nil,FILE_LIST_FROM_ARGS
+          Log.log.debug("getting file list as parameters")
+          # get remaining arguments
+          file_list=@opt_mgr.get_next_argument("source file list",:multiple)
+          raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
+        when FILE_LIST_FROM_TRANSFER_SPEC
+          Log.log.debug("assume list provided in transfer spec")
+          raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil?
+          # here we assume check of sources is made in transfer agent
+          return @transfer_paths
+        when Array
+          Log.log.debug("getting file list as extended value")
+          raise CliBadArgument,"sources must be a Array of String" if !file_list.select{|f|!f.is_a?(String)}.empty?
+        else
+          raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
+        end
+        # here, file_list is an Array or String
+        if !@transfer_paths.nil?
+          Log.log.warn("--sources overrides paths from --ts")
+        end
+        case @opt_mgr.get_option(:src_type,:mandatory)
+        when :list
+          # when providing a list, just specify source
+          @transfer_paths=file_list.map{|i|{'source'=>i}}
+        when :pair
+          raise CliBadArgument,"whe using pair, provide even number of paths: #{file_list.length}" unless file_list.length.even?
+          @transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
+        else raise "ERROR"
+        end
+        Log.log.debug("paths=#{@transfer_paths}")
+        return @transfer_paths
+      end
+
+      # start a transfer and wait for completion, plugins shall use this method
+      # @param transfer_spec
+      # @param options specific options for the transfer_agent
+      # options[:src] specifies how destination_root is set (how transfer spec was generated)
+      # other options are carried to specific agent
+      def start(transfer_spec,options)
+        # check parameters
+        raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
+        raise "options must be hash" unless options.is_a?(Hash)
+        # process :src option
+        case transfer_spec['direction']
+        when 'receive'
+          # init default if required in any case
+          @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
+        when 'send'
+          case options[:src]
+          when :direct
+            # init default if required
+            @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
+          when :node_gen3
+            # in that case, destination is set in return by application (API/upload_setup)
+            # but to_folder was used in initial API call
+            @transfer_spec_cmdline.delete('destination_root')
+          when :node_gen4
+            @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.has_key?('destination_root_id')
+          else
+            raise StandardError,"InternalError: unsupported value: #{options[:src]}"
+          end
+        end
+
+        # only used here
+        options.delete(:src)
+
+        # update command line paths, unless destination already has one
+        @transfer_spec_cmdline['paths']=transfer_spec['paths'] || ts_source_paths
+
+        transfer_spec.merge!(@transfer_spec_cmdline)
+        # create transfer agent
+        self.set_agent_by_options
+        Log.log.debug("transfer agent is a #{@agent.class}")
+        @agent.start_transfer(transfer_spec,options)
+        result=@agent.wait_for_transfers_completion
+        @progress_listener.reset
+        Fasp::Manager.validate_status_list(result)
+        return result
+      end
+
+      # @return :success if all sessions statuses returned by "start" are success
+      # else return the first error exception object
+      def self.session_status(statuses)
+        error_statuses=statuses.select{|i|!i.eql?(:success)}
+        return :success if error_statuses.empty?
+        return error_statuses.first
+      end
+
+      # shut down if agent requires it
+      def shutdown
+        @agent.shutdown if @agent.respond_to?(:shutdown)
+      end
+
+    end
+  end
+end

data/lib/aspera/cli/version.rb

@@ -0,0 +1,5 @@
+module Aspera
+  module Cli
+    VERSION = "4.0.0.pre1"
+  end
+end

data/lib/aspera/colors.rb

@@ -0,0 +1,39 @@
+# simple vt100 colors
+class String
+  private
+  def self.vtcmd(code);"\e[#{code}m";end
+  # see https://en.wikipedia.org/wiki/ANSI_escape_code
+  # symbol is the method name added to String
+  # it adds control chars to set color (and reset at the end).
+  VTSTYLES = {
+    :bold=>1,
+    :italic=>3,
+    :underline=>4,
+    :blink=>5,
+    :reverse_color=>7,
+    :black=>30,
+    :red=>31,
+    :green=>32,
+    :brown=>33,
+    :blue=>34,
+    :magenta=>35,
+    :cyan=>36,
+    :gray=>37,
+    :bg_black=>40,
+    :bg_red=>41,
+    :bg_green=>42,
+    :bg_brown=>43,
+    :bg_blue=>44,
+    :bg_magenta=>45,
+    :bg_cyan=>46,
+    :bg_gray=>47,
+  }
+  private_constant :VTSTYLES
+  # defines methods to String, one per entry in VTSTYLES
+  VTSTYLES.each do |name,code|
+    begin_seq=vtcmd(code)
+    end_seq=vtcmd((code >= 10) ? 0 : code+20+(code.eql?(1)?1:0))
+    define_method(name){"#{begin_seq}#{self}#{end_seq}"}
+    public name
+  end
+end

data/lib/aspera/command_line_builder.rb

@@ -0,0 +1,137 @@
+module Aspera
+  # helper class to build command line from a parameter list (key-value hash)
+  # constructor takes hash: { 'param1':'value1', ...}
+  # process_param is called repeatedly with all known parameters
+  # add_env_args is called to get resulting param list and env var (also checks that all params were used)
+  class CommandLineBuilder
+
+    private
+    # default value for command line based on option name
+    def switch_name(param_name,options)
+      return options[:option_switch] if options.has_key?(:option_switch)
+      return '--'+param_name.to_s.gsub('_','-')
+    end
+
+    def env_name(param_name,options)
+      return options[:variable]
+    end
+
+    public
+
+    BOOLEAN_CLASSES=[TrueClass,FalseClass]
+
+    # @param param_hash
+    def initialize(param_hash,params_definition)
+      @param_hash=param_hash  # keep reference so that it can be modified by caller before calling `process_params`
+      @params_definition=params_definition
+      @result_env={}
+      @result_args=[]
+      @used_param_names=[]
+    end
+
+    def warn_unrecognized_params
+      # warn about non translated arguments
+      @param_hash.each_pair{|key,val|Log.log.warn("unrecognized parameter: #{key} = \"#{val}\"") if !@used_param_names.include?(key)}
+    end
+
+    # adds keys :env :args with resulting values after processing
+    # warns if some parameters were not used
+    def add_env_args(env,args)
+      Log.log.debug("ENV=#{@result_env}, ARGS=#{@result_args}")
+      warn_unrecognized_params
+      env.merge!(@result_env)
+      args.push(*@result_args)
+      return nil
+    end
+
+    # transform yes/no to trye/false
+    def self.yes_to_true(value)
+      case value
+      when 'yes'; return true
+      when 'no'; return false
+      end
+      raise "unsupported value: #{value}"
+    end
+
+    # add options directly to ascp command line
+    def add_command_line_options(options)
+      return if options.nil?
+      options.each{|o|@result_args.push(o.to_s)}
+    end
+
+    def process_params
+      @params_definition.keys.each do |k|
+        process_param(k)
+      end
+    end
+
+    # Process a parameter from transfer specification and generate command line param or env var
+    # @param param_name : key in transfer spec
+    # @param action : type of processing: ignore getvalue envvar opt_without_arg opt_with_arg defer
+    # @param options : options for type
+    def process_param(param_name,action=nil)
+      options=@params_definition[param_name]
+      action=options[:type] if action.nil?
+      # should not happen
+      raise "Internal error: ask processing of param #{param_name}" if options.nil?
+      # by default : not mandatory
+      options[:mandatory]||=false
+      if options.has_key?(:accepted_types)
+        # single type is placed in array
+        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
+      else
+        # by default : string, unless it's without arg
+        options[:accepted_types]=action.eql?(:opt_without_arg) ? BOOLEAN_CLASSES : [String]
+      end
+      # check mandatory parameter (nil is valid value)
+      raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
+      parameter_value=@param_hash[param_name]
+      parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      # check provided type
+      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or options[:accepted_types].inject(false){|m,v|m or parameter_value.is_a?(v)}
+      @used_param_names.push(param_name) unless action.eql?(:defer)
+
+      # process only non-nil values
+      return nil if parameter_value.nil?
+
+      if options.has_key?(:translate_values)
+        # translate using conversion table
+        new_value=options[:translate_values][parameter_value]
+        raise "unsupported value: #{parameter_value}" if new_value.nil?
+        parameter_value=new_value
+      end
+      raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
+      if options[:encode]
+        newvalue=options[:encode].call(parameter_value)
+        raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
+        parameter_value=newvalue
+      end
+
+      case action
+      when :ignore,:defer # ignore this parameter or process later
+        return
+      when :get_value # just get value
+        return parameter_value
+      when :envvar # set in env var
+        # define ascp parameter in env var from transfer spec
+        @result_env[env_name(param_name,options)] = parameter_value
+      when :opt_without_arg # if present and true : just add option without value
+        add_param=false
+        case parameter_value
+        when false# nothing to put on command line, no creation by default
+        when true; add_param=true
+        else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
+        end
+        add_param=!add_param if options[:add_on_false]
+        add_command_line_options([switch_name(param_name,options)]) if add_param
+      when :opt_with_arg # transform into command line option with value
+        #parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
+        parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
+        # if transfer_spec value is an array, applies option many times
+        parameter_value.each{|v|add_command_line_options([switch_name(param_name,options),v])}
+      else
+        raise "Error"
+      end
+    end
+  end
+end