aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/cli/main.rb

@@ -4,82 +4,96 @@ require 'aspera/cli/plugins/config'
 require 'aspera/cli/extended_value'
 require 'aspera/cli/transfer_agent'
 require 'aspera/cli/version'
+require 'aspera/fasp/error'
 require 'aspera/open_application'
 require 'aspera/temp_file_manager'
 require 'aspera/persistency_folder'
 require 'aspera/log'
 require 'aspera/rest'
 require 'aspera/nagios'
-require 'aspera/secrets'
 
 module Aspera
   module Cli
     # The main CLI class
     class Main
 
-      attr_reader :plugin_env
       private
-      # name of application, also foldername where config is stored
+      # name of application, also used as foldername where config is stored
       PROGRAM_NAME = 'ascli'
+      # name of the containing gem, same as in <gem name>.gemspec
       GEM_NAME = 'aspera-cli'
-      VERBOSE_LEVELS=[:normal,:minimal,:quiet]
-
-      private_constant :PROGRAM_NAME,:GEM_NAME,:VERBOSE_LEVELS
+      HELP_URL = "http://www.rubydoc.info/gems/#{GEM_NAME}"
+      GEM_URL  = "https://rubygems.org/gems/#{GEM_NAME}"
+      SRC_URL  = 'https://github.com/IBM/aspera-cli'
+      # store transfer result using this key and use result_transfer_multiple
+      STATUS_FIELD = 'status'
 
+      private_constant :PROGRAM_NAME,:GEM_NAME,:HELP_URL,:GEM_URL
       # =============================================================
       # Parameter handlers
       #
-
-      def option_insecure; Rest.insecure ; end
-
-      def option_insecure=(value); Rest.insecure = value; end
-
+      attr_accessor :option_insecure, :option_http_options
       def option_ui; OpenApplication.instance.url_method; end
 
       def option_ui=(value); OpenApplication.instance.url_method=value; end
 
+      # called everytime a new REST HTTP session is opened
+      # @param http [Net::HTTP] the newly created http session object
+      def set_http_parameters(http)
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @option_insecure
+        http.set_debug_output($stdout) if @option_rest_debug
+        raise "http_options expects Hash" unless @option_http_options.is_a?(Hash)
+        @option_http_options.each do |k,v|
+          method="#{k}=".to_sym
+          # check if accessor is a method of Net::HTTP
+          # continue_timeout= read_timeout= write_timeout=
+          if http.respond_to?(method)
+            http.send(method,v)
+          else
+            Log.log.error("no such attribute: #{k}")
+          end
+        end
+      end
+
       # minimum initialization
       def initialize(argv)
-        # first thing : manage debug level (allows debugging or option parser)
+        # first thing : manage debug level (allows debugging of option parser)
         early_debug_setup(argv)
+        # compare $0 with expected name
         current_prog_name=File.basename($PROGRAM_NAME)
         unless current_prog_name.eql?(PROGRAM_NAME)
           @plugin_env[:formater].display_message(:error,"#{"WARNING".bg_red.blink.gray} Please use '#{PROGRAM_NAME}' instead of '#{current_prog_name}', '#{current_prog_name}' will be removed in a future version")
         end
-        # overriding parameters on transfer spec
         @option_help=false
         @bash_completion=false
         @option_show_config=false
+        @option_insecure=false
+        @option_rest_debug=false
+        @option_http_options={}
+        # environment provided to plugin for various capabilities
         @plugin_env={}
-        @help_url='http://www.rubydoc.info/gems/'+GEM_NAME
-        @gem_url='https://rubygems.org/gems/'+GEM_NAME
-        # give command line arguments to option manager (no parsing)
-        app_main_folder=ENV[conf_dir_env_var]
-        # if env var undefined or empty
-        if app_main_folder.nil? or app_main_folder.empty?
-          user_home_folder=Dir.home
-          raise CliError,"Home folder does not exist: #{user_home_folder}. Check your user environment or use #{conf_dir_env_var}." unless Dir.exist?(user_home_folder)
-          app_main_folder=File.join(user_home_folder,Plugins::Config::ASPERA_HOME_FOLDER_NAME,PROGRAM_NAME)
-        end
-        @plugin_env[:options]=@opt_mgr=Manager.new(PROGRAM_NAME,argv,app_banner())
+        # give command line arguments to option manager
+        @plugin_env[:options]=@opt_mgr=Manager.new(PROGRAM_NAME,argv)
+        # formatter adds options
         @plugin_env[:formater]=Formater.new(@plugin_env[:options])
         Rest.user_agent=PROGRAM_NAME
-        # must override help methods before parser called (in other constructors)
+        Rest.session_cb=lambda {|http| set_http_parameters(http)}
+        # declare and parse global options
         init_global_options()
-        # secret manager
-        @plugin_env[:secret]=Aspera::Secrets.new
-        # the Config plugin adds the @preset parser
-        @plugin_env[:config]=Plugins::Config.new(@plugin_env,PROGRAM_NAME,@help_url,Aspera::Cli::VERSION,app_main_folder)
+        # the Config plugin adds the @preset parser, so declare before TransferAgent which may use it
+        @plugin_env[:config]=Plugins::Config.new(@plugin_env,PROGRAM_NAME,HELP_URL,Aspera::Cli::VERSION,app_main_folder)
         # the TransferAgent plugin may use the @preset parser
-        @plugin_env[:transfer]=TransferAgent.new(@plugin_env)
-        Log.log.debug('created plugin env'.red)
-        # set application folder for modules
+        @plugin_env[:transfer]=TransferAgent.new(@plugin_env[:options],@plugin_env[:config])
+        # data persistency
         @plugin_env[:persistency]=PersistencyFolder.new(File.join(@plugin_env[:config].main_folder,'persist_store'))
+        Log.log.debug('plugin env created'.red)
         Oauth.persist_mgr=@plugin_env[:persistency]
         Fasp::Parameters.file_list_folder=File.join(@plugin_env[:config].main_folder,'filelists')
         Aspera::RestErrorAnalyzer.instance.log_file=File.join(@plugin_env[:config].main_folder,'rest_exceptions.log')
         # register aspera REST call error handlers
         Aspera::RestErrorsAspera.registerHandlers
+        # set banner when all environment is created so that additional extended value modifiers are known, e.g. @preset
+        @opt_mgr.parser.banner=app_banner
       end
 
       def app_banner
@@ -88,18 +102,19 @@ module Aspera
         banner << "\t#{PROGRAM_NAME} COMMANDS [OPTIONS] [ARGS]\n"
         banner << "\nDESCRIPTION\n"
         banner << "\tUse Aspera application to perform operations on command line.\n"
-        banner << "\tDocumentation and examples: #{@gem_url}\n"
+        banner << "\tDocumentation and examples: #{GEM_URL}\n"
         banner << "\texecute: #{PROGRAM_NAME} conf doc\n"
-        banner << "\tor visit: #{@help_url}\n"
+        banner << "\tor visit: #{HELP_URL}\n"
+        banner << "\tsource repo: #{SRC_URL}\n"
         banner << "\nENVIRONMENT VARIABLES\n"
         banner << "\t#{conf_dir_env_var}  config folder, default: $HOME/#{Plugins::Config::ASPERA_HOME_FOLDER_NAME}/#{PROGRAM_NAME}\n"
-        banner << "\t#any option can be set as an environment variable, refer to the manual\n"
+        banner << "\tany option can be set as an environment variable, refer to the manual\n"
         banner << "\nCOMMANDS\n"
         banner << "\tTo list first level commands, execute: #{PROGRAM_NAME}\n"
         banner << "\tNote that commands can be written shortened (provided it is unique).\n"
         banner << "\nOPTIONS\n"
         banner << "\tOptions begin with a '-' (minus), and value is provided on command line.\n"
-        banner << "\tSpecial values are supported beginning with special prefix, like: #{ExtendedValue.instance.modifiers.map{|m|"@#{m}:"}.join(' ')}.\n"
+        banner << "\tSpecial values are supported beginning with special prefix @pfx:, where pfx is one of:\n\t#{ExtendedValue.instance.modifiers.map{|m|m.to_s}.join(', ')}\n"
         banner << "\tDates format is 'DD-MM-YY HH:MM:SS', or 'now' or '-<num>h'\n\n"
         banner << "ARGS\n"
         banner << "\tSome commands require mandatory arguments, e.g. a path.\n"
@@ -111,7 +126,7 @@ module Aspera
         @opt_mgr.add_opt_switch(:help,"-h","Show this message.") { @option_help=true }
         @opt_mgr.add_opt_switch(:bash_comp,"generate bash completion for command") { @bash_completion=true }
         @opt_mgr.add_opt_switch(:show_config, "Display parameters used for the provided action.") { @option_show_config=true }
-        @opt_mgr.add_opt_switch(:rest_debug,"-r","more debug for HTTP calls") { Rest.debug=true }
+        @opt_mgr.add_opt_switch(:rest_debug,"-r","more debug for HTTP calls") { @option_rest_debug=true }
         @opt_mgr.add_opt_switch(:version,'-v','display version') { @plugin_env[:formater].display_message(:data,Aspera::Cli::VERSION);Process.exit(0) }
         @opt_mgr.add_opt_switch(:warnings,'-w','check for language warnings') { $VERBOSE=true }
         # handler must be set before declaration
@@ -119,13 +134,17 @@ module Aspera
         @opt_mgr.set_obj_attr(:logger,Log.instance,:logger_type)
         @opt_mgr.set_obj_attr(:insecure,self,:option_insecure,:no)
         @opt_mgr.set_obj_attr(:ui,self,:option_ui)
+        @opt_mgr.set_obj_attr(:http_options,self,:option_http_options)
+        @opt_mgr.set_obj_attr(:log_passwords,Log.instance,:log_passwords)
         @opt_mgr.add_opt_list(:ui,OpenApplication.user_interfaces,'method to start browser')
         @opt_mgr.add_opt_list(:log_level,Log.levels,"Log level")
         @opt_mgr.add_opt_list(:logger,Log.logtypes,"log method")
         @opt_mgr.add_opt_simple(:lock_port,"prevent dual execution of a command, e.g. in cron")
         @opt_mgr.add_opt_simple(:query,"additional filter for API calls (extended value) (some commands)")
+        @opt_mgr.add_opt_simple(:http_options,"options for http socket (extended value)")
         @opt_mgr.add_opt_boolean(:insecure,"do not validate HTTPS certificate")
         @opt_mgr.add_opt_boolean(:once_only,"process only new items (some commands)")
+        @opt_mgr.add_opt_boolean(:log_passwords,"show passwords in logs")
         @opt_mgr.set_option(:ui,OpenApplication.default_gui_mode)
         @opt_mgr.set_option(:once_only,:false)
         # parse declared options
@@ -141,7 +160,7 @@ module Aspera
         require @plugin_env[:config].plugins[plugin_name_sym][:require_stanza]
         # load default params only if no param already loaded before plugin instanciation
         env[:config].add_plugin_default_preset(plugin_name_sym)
-        command_plugin=Plugins::Config.plugin_new(plugin_name_sym,env)
+        command_plugin=Plugins::Config.plugin_class(plugin_name_sym).new(env)
         Log.log.debug("got #{command_plugin.class}")
         # TODO: check that ancestor is Plugin?
         return command_plugin
@@ -177,10 +196,11 @@ module Aspera
             # override main option parser with a brand new, to avoid having global options
             plugin_env=@plugin_env.clone
             plugin_env[:man_only]=true
-            plugin_env[:options]=Manager.new(PROGRAM_NAME,[],'')
+            plugin_env[:options]=Manager.new(PROGRAM_NAME)
+            plugin_env[:options].parser.banner='' # remove default banner
             get_plugin_instance_with_options(plugin_name_sym,plugin_env)
             # display generated help for plugin options
-            @plugin_env[:formater].display_message(:error,plugin_env[:options].parser.to_s)
+            @plugin_env[:formater].display_message(:error,plugin_env[:options].parser.help)
           end
         end
         Process.exit(0)
@@ -188,10 +208,24 @@ module Aspera
 
       protected
 
+      # env var name to override the app's main folder
+      # default main folder is $HOME/<vendor main app folder>/<program name>
       def conf_dir_env_var
         return "#{PROGRAM_NAME}_home".upcase
       end
 
+      def app_main_folder
+        # find out application main folder
+        app_folder=ENV[conf_dir_env_var]
+        # if env var undefined or empty
+        if app_folder.nil? or app_folder.empty?
+          user_home_folder=Dir.home
+          raise CliError,"Home folder does not exist: #{user_home_folder}. Check your user environment or use #{conf_dir_env_var}." unless Dir.exist?(user_home_folder)
+          app_folder=File.join(user_home_folder,Plugins::Config::ASPERA_HOME_FOLDER_NAME,PROGRAM_NAME)
+        end
+        return app_folder
+      end
+
       # early debug for parser
       # Note: does not accept shortcuts
       def early_debug_setup(argv)
@@ -219,19 +253,35 @@ module Aspera
         return Main.result_nothing
       end
 
+      # used when one command executes several transfer jobs (each job being possibly multi session)
+      # @param status_table [Array] [{STATUS_FIELD=>[status array],...},...]
+      # @return a status object suitable as command result
+      # each element has a key STATUS_FIELD which contains the result of possibly multiple sessions
+      def self.result_transfer_multiple(status_table)
+        global_status=:success
+        # transform status array into string and find if there was problem
+        status_table.each do |item|
+          worst=TransferAgent.session_status(item[STATUS_FIELD])
+          global_status=worst unless worst.eql?(:success)
+          item[STATUS_FIELD]=item[STATUS_FIELD].map{|i|i.to_s}.join(',')
+        end
+        raise global_status unless global_status.eql?(:success)
+        return {:type=>:object_list,:data=>status_table}
+      end
+
       # this is the main function called by initial script just after constructor
       def process_command_line
         Log.log.debug('process_command_line')
+        # catch exception information , if any
         exception_info=nil
+        # false if command shall not be executed ("once_only")
+        execute_command=true
         begin
           # find plugins, shall be after parse! ?
           @plugin_env[:config].add_plugins_from_lookup_folders
           # help requested without command ? (plugins must be known here)
           exit_with_usage(true) if @option_help and @opt_mgr.command_or_arg_empty?
           generate_bash_completion if @bash_completion
-          # load global default options and process
-          @plugin_env[:config].add_plugin_default_preset(Plugins::Config::CONF_GLOBAL_SYM)
-          @opt_mgr.parse_options!
           @plugin_env[:config].periodic_check_newer_gem_version
           if @option_show_config and @opt_mgr.command_or_arg_empty?
             command_sym=Plugins::Config::CONF_PLUGIN_SYM
@@ -264,11 +314,12 @@ module Aspera
               Log.log.debug("Opening lock port #{lock_port.to_i}")
               @tcp_server=TCPServer.new('127.0.0.1',lock_port.to_i)
             rescue => e
-              raise CliError,"Another instance is already running (#{e.message})."
+              execute_command=false
+              Log.log.warn("Another instance is already running (#{e.message}).")
             end
           end
-          # execute and display
-          @plugin_env[:formater].display_results(command_plugin.execute_action)
+          # execute and display (if not exclusive execution)
+          @plugin_env[:formater].display_results(command_plugin.execute_action) if execute_command
           # finish
           @plugin_env[:transfer].shutdown
         rescue CliBadArgument => e;          exception_info=[e,'Argument',:usage]
@@ -286,10 +337,17 @@ module Aspera
         unless exception_info.nil?
           @plugin_env[:formater].display_message(:error,"ERROR:".bg_red.gray.blink+" "+exception_info[1]+": "+exception_info[0].message)
           @plugin_env[:formater].display_message(:error,"Use '-h' option to get help.") if exception_info[2].eql?(:usage)
+          if exception_info.first.is_a?(Fasp::Error) and exception_info.first.message.eql?('Remote host is not who we expected')
+            @plugin_env[:formater].display_message(:error,"For this specific error, refer to:\n#{SRC_URL}#error-remote-host-is-not-who-we-expected\nAdd this to arguments:\n--ts=@json:'{\"sshfp\":null}'")
+          end
         end
         # 2- processing of command not processed (due to exception or bad command line)
-        @opt_mgr.final_errors.each do |msg|
-          @plugin_env[:formater].display_message(:error,"ERROR:".bg_red.gray.blink+" Argument: "+msg)
+        if execute_command
+          @opt_mgr.final_errors.each do |msg|
+            @plugin_env[:formater].display_message(:error,"ERROR:".bg_red.gray.blink+" Argument: "+msg)
+            # add code as exception if there is not already an error
+            exception_info=[Exception.new(msg),'UnusedArg'] if exception_info.nil?
+          end
         end
         # 3- in case of error, fail the process status
         unless exception_info.nil?

data/lib/aspera/cli/manager.rb

@@ -79,7 +79,7 @@ module Aspera
       attr_accessor :ask_missing_optional
 
       #
-      def initialize(program_name,argv,app_banner)
+      def initialize(program_name,argv=nil)
         # command line values not starting with '-'
         @unprocessed_cmd_line_arguments=[]
         # command line values starting with '-'
@@ -98,7 +98,6 @@ module Aspera
         # Note: was initially inherited but it is prefered to have specific methods
         @parser=OptionParser.new
         @parser.program_name=program_name
-        @parser.banner=app_banner
         # options can also be provided by env vars : --param-name -> ASLMCLI_PARAM_NAME
         env_prefix=program_name.upcase+OPTION_SEP_NAME
         ENV.each do |k,v|
@@ -107,29 +106,29 @@ module Aspera
           end
         end
         Log.log.debug("env=#{@unprocessed_env}".red)
-        # banner is empty when help is generated for every plugin
-        unless app_banner.empty?
-          @parser.separator("")
-          @parser.separator("OPTIONS: global")
+        @unprocessed_cmd_line_options=[]
+        @unprocessed_cmd_line_arguments=[]
+        # argv is nil when help is generated for every plugin
+        unless argv.nil?
+          @parser.separator('')
+          @parser.separator('OPTIONS: global')
           self.set_obj_attr(:interactive,self,:ask_missing_mandatory)
           self.set_obj_attr(:ask_options,self,:ask_missing_optional)
-          self.add_opt_boolean(:interactive,"use interactive input of missing params")
-          self.add_opt_boolean(:ask_options,"ask even optional options")
+          self.add_opt_boolean(:interactive,'use interactive input of missing params')
+          self.add_opt_boolean(:ask_options,'ask even optional options')
           self.parse_options!
-        end
-        @unprocessed_cmd_line_options=[]
-        @unprocessed_cmd_line_arguments=[]
-        process_options=true
-        while !argv.empty?
-          value=argv.shift
-          if process_options and value.start_with?('-')
-            if value.eql?('--')
-              process_options=false
+          process_options=true
+          while !argv.empty?
+            value=argv.shift
+            if process_options and value.start_with?('-')
+              if value.eql?('--')
+                process_options=false
+              else
+                @unprocessed_cmd_line_options.push(value)
+              end
             else
-              @unprocessed_cmd_line_options.push(value)
+              @unprocessed_cmd_line_arguments.push(value)
             end
-          else
-            @unprocessed_cmd_line_arguments.push(value)
           end
         end
         @initial_cli_options=@unprocessed_cmd_line_options.dup

data/lib/aspera/cli/plugin.rb

@@ -2,10 +2,15 @@ module Aspera
   module Cli
     # base class for plugins modules
     class Plugin
+      # operation without id
       GLOBAL_OPS=[:create,:list]
+      # operation on specific instance
       INSTANCE_OPS=[:modify,:delete,:show]
       ALL_OPS=[GLOBAL_OPS,INSTANCE_OPS].flatten
-      #private_constant :GLOBAL_OPS,:INSTANCE_OPS,:ALL_OPS
+      # max number of items for list command
+      MAX_ITEMS='max'
+      # max number of pages for list command
+      MAX_PAGES='pmax'
 
       @@done=false
 
@@ -28,10 +33,18 @@ module Aspera
         end
       end
 
-      def entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default=nil,subkey=false)
+      # must be called AFTER the instance action
+      def instance_identifier()
+        res_id=self.options.get_option(:id)
+        res_id=self.options.get_next_argument('identifier') if res_id.nil?
+        return res_id
+      end
+
+      def entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default=nil,use_subkey=false)
+        raise "not id" unless :id.eql?(id_symb)
         if INSTANCE_OPS.include?(command)
           begin
-            one_res_id=self.options.get_option(id_symb,:mandatory)
+            one_res_id=instance_identifier()
           rescue => e
             raise e if id_default.nil?
             one_res_id=id_default
@@ -54,10 +67,11 @@ module Aspera
         when :list
           resp=rest_api.read(res_class_path,parameters)
           data=resp[:data]
+          # TODO: not generic : which application is this for ?
           if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
-            data=resp[:data][res_class_path]
+            data=data[res_class_path]
           end
-          data=data[res_class_path] if subkey
+          data=data[res_class_path] if use_subkey
           return {:type => :object_list, :data=>data, :fields=>display_fields}
         when :modify
           property=self.options.get_option(:property,:optional)
@@ -74,12 +88,13 @@ module Aspera
       end
 
       # implement generic rest operations on given resource path
-      def entity_action(rest_api,res_class_path,display_fields,id_symb,id_default=nil,subkey=false)
+      def entity_action(rest_api,res_class_path,display_fields,id_symb,id_default=nil,use_subkey=false)
         #res_name=res_class_path.gsub(%r{^.*/},'').gsub(%r{s$},'').gsub('_',' ')
         command=self.options.get_next_command(ALL_OPS)
-        return entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default,subkey)
+        return entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default,use_subkey)
       end
 
+      # shortcuts for plugin environment
       def options; return @agents[:options];end
 
       def transfer; return @agents[:transfer];end