aspera-cli 4.7.0 → 4.9.0

data/lib/aspera/cli/formater.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+
+require 'aspera/secret_hider'
 require 'terminal-table'
 require 'yaml'
 require 'pp'
@@ -7,75 +9,78 @@ module Aspera
   module Cli
     # Take care of output
     class Formater
-      FIELDS_ALL='ALL'
-      FIELDS_DEFAULT='DEF'
+      FIELDS_ALL = 'ALL'
+      FIELDS_DEFAULT = 'DEF'
+      CSV_RECORD_SEPARATOR = "\n"
+      CSV_FIELD_SEPARATOR = ','
       # supported output formats
-      DISPLAY_FORMATS=%i[table ruby json jsonpp yaml csv nagios]
+      DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table csv].freeze
       # user output levels
-      DISPLAY_LEVELS=[:info,:data,:error]
-      CSV_RECORD_SEPARATOR="\n"
-      CSV_FIELD_SEPARATOR=','
-      HIDDEN_PASSWORD='🔑'
-      SECRET_KEYWORDS=%w[password secret private_key]
+      DISPLAY_LEVELS = %i[info data error].freeze
+      CONF_OVERVIEW_KEYS=%w[config parameter value].freeze
 
-      private_constant :FIELDS_ALL,:FIELDS_DEFAULT,:DISPLAY_FORMATS,:DISPLAY_LEVELS,:CSV_RECORD_SEPARATOR,:CSV_FIELD_SEPARATOR,:HIDDEN_PASSWORD
+      private_constant :FIELDS_ALL,:FIELDS_DEFAULT,:DISPLAY_FORMATS,:DISPLAY_LEVELS,:CSV_RECORD_SEPARATOR,:CSV_FIELD_SEPARATOR,
+        :CONF_OVERVIEW_KEYS
 
-      class<<self
-        # @param source [Hash] hash to modify
-        # @param keep_last [bool]
-        def flatten_object(source,keep_last)
-          newval={}
-          flatten_sub_hash_rec(source,keep_last,'',newval)
-          source.clear
-          source.merge!(newval)
+      class << self
+        # special for Aspera on Cloud display node
+        # {"param" => [{"name"=>"foo","value"=>"bar"}]} will be expanded to {"param.foo" : "bar"}
+        def flatten_name_value_list(hash)
+          hash.keys.each do |k|
+            v = hash[k]
+            next unless v.is_a?(Array) && v.map(&:class).uniq.eql?([Hash]) && v.map(&:keys).flatten.sort.uniq.eql?(%w[name value])
+            v.each do |pair|
+              hash["#{k}.#{pair['name']}"] = pair['value']
+            end
+            hash.delete(k)
+          end
+        end
+
+        def flatten_config_overview(conffile)
+          r = []
+          conffile.each do |config,preset|
+            preset.each do |parameter,value|
+              r.push(CONF_OVERVIEW_KEYS.zip([config,parameter,value]).to_h)
+            end
+          end
+          return r
+        end
+
+        def simple_hash?(h)
+          !(h.values.any?{|v|[Hash,Array].any?{|c|v.is_a?(c)}})
         end
 
         # recursive function to modify a hash
         # @param source [Hash] to be modified
-        # @param keep_last [bool] truer if last level is not
+        # @param expand_last [bool] truer if last level is not
+        # @param result [Hash] new hash flattened
         # @param prefix [String] true if last level is not
-        # @param dest [Hash] new hash flattened
-        def flatten_sub_hash_rec(source,keep_last,prefix,dest)
-          #is_simple_hash=source.is_a?(Hash) and source.values.inject(true){|m,v| xxx=!v.respond_to?(:each) and m;puts("->#{xxx}>#{v.respond_to?(:each)} #{v}-");xxx}
-          is_simple_hash=false
-          Log.log.debug("(#{keep_last})[#{is_simple_hash}] -#{source.values}- \n-#{source}-")
-          return source if keep_last && is_simple_hash
+        def flattened_object(source,result: {},prefix: '',expand_last: false)
+          Log.log.debug("(#{expand_last})[#{simple_hash?(source)}] -#{source.values}- \n-#{source}-")
           source.each do |k,v|
-            if v.is_a?(Hash) && (!keep_last || !is_simple_hash)
-              flatten_sub_hash_rec(v,keep_last,prefix+k.to_s+'.',dest)
+            if v.is_a?(Hash) && !(expand_last && simple_hash?(v))
+              flattened_object(v,result: result,prefix: prefix + k.to_s + '.',expand_last: expand_last)
             else
-              dest[prefix+k.to_s]=v
+              result[prefix + k.to_s] = v
             end
           end
-          return nil
-        end
-
-        # special for Aspera on Cloud display node
-        # {"param" => [{"name"=>"foo","value"=>"bar"}]} will be expanded to {"param.foo" : "bar"}
-        def flatten_name_value_list(hash)
-          hash.keys.each do |k|
-            v=hash[k]
-            next unless v.is_a?(Array) && v.map(&:class).uniq.eql?([Hash]) && v.map(&:keys).flatten.sort.uniq.eql?(['name', 'value'])
-            v.each do |pair|
-              hash["#{k}.#{pair['name']}"]=pair['value']
-            end
-            hash.delete(k)
-          end
+          return result
         end
       end
 
-      attr_accessor :option_flat_hash,:option_transpose_single,:option_format,:option_display,:option_fields,:option_table_style,:option_select,:option_show_secrets
+      attr_accessor :option_flat_hash,:option_transpose_single,:option_format,:option_display,:option_fields,:option_table_style,
+        :option_select,:option_show_secrets
 
       # adds options but does not parse
       def initialize(opt_mgr)
-        @option_format=:table
-        @option_display=:info
-        @option_fields=FIELDS_DEFAULT
-        @option_select=nil
-        @option_table_style=':.:'
-        @option_flat_hash=true
-        @option_transpose_single=true
-        @option_show_secrets=true
+        @option_format = :table
+        @option_display = :info
+        @option_fields = FIELDS_DEFAULT
+        @option_select = nil
+        @option_table_style = ':.:'
+        @option_flat_hash = true
+        @option_transpose_single = true
+        @option_show_secrets = false
         opt_mgr.set_obj_attr(:format,self,:option_format)
         opt_mgr.set_obj_attr(:display,self,:option_display)
         opt_mgr.set_obj_attr(:fields,self,:option_fields)
@@ -95,10 +100,13 @@ module Aspera
       end
 
       # main output method
+      # data: for requested data, not displayed if level==error
+      # info: additional info, displayed if level==info
+      # error: always displayed on stderr
       def display_message(message_level,message)
         case message_level
-        when :info then $stdout.puts(message) if @option_display.eql?(:info)
         when :data then $stdout.puts(message) unless @option_display.eql?(:error)
+        when :info then $stdout.puts(message) if @option_display.eql?(:info)
         when :error then $stderr.puts(message)
         else raise "wrong message_level:#{message_level}"
         end
@@ -112,7 +120,7 @@ module Aspera
         unless results[:fields].nil?
           raise "internal error: [fields] must be Array, not #{results[:fields].class}" unless results[:fields].is_a?(Array)
           if results[:fields].first.eql?(:all_but) && !table_rows_hash_val.empty?
-            filter=results[:fields][1..-1]
+            filter = results[:fields][1..-1]
             return table_rows_hash_val.first.keys.reject{|i|filter.include?(i)}
           end
           return results[:fields]
@@ -124,39 +132,27 @@ module Aspera
       def result_all_fields(_results,table_rows_hash_val)
         raise 'internal error: must be array' unless table_rows_hash_val.is_a?(Array)
         # get the list of all column names used in all lines, not just frst one, as all lines may have different columns
-        return table_rows_hash_val.each_with_object({}){|v,m|v.keys.each{|c|m[c]=true};}.keys
-      end
-
-      def deep_remove_secret(obj)
-        return if @option_show_secrets
-        case obj
-        when Array then obj.each{|i|deep_remove_secret(i)}
-        when Hash
-          obj.keys.each do |k|
-            next if %w[show_secrets log_secrets].any?{|kw|kw.eql?(k)}
-            if k.is_a?(String) && SECRET_KEYWORDS.any?{|kw|k.include?(kw)}
-              obj[k]=HIDDEN_PASSWORD
-            elsif obj[k].is_a?(Hash)
-              deep_remove_secret(obj[k])
-            end
-          end
-        end
+        return table_rows_hash_val.each_with_object({}){|v,m|v.keys.each{|c|m[c] = true};}.keys
       end
 
       # this method displays the results, especially the table format
       def display_results(results)
         raise "INTERNAL ERROR, result must be Hash (got: #{results.class}: #{results})" unless results.is_a?(Hash)
         raise 'INTERNAL ERROR, result must have type' unless results.has_key?(:type)
-        raise 'INTERNAL ERROR, result must have data' unless results.has_key?(:data) || [:empty,:nothing].include?(results[:type])
-        res_data=results[:data]
-        deep_remove_secret(res_data)
+        raise 'INTERNAL ERROR, result must have data' unless results.has_key?(:data) || %i[empty nothing].include?(results[:type])
+        res_data = results[:data]
+        # for config overvuew, it is name and value
+        is_config_overview = res_data.is_a?(Array) && !res_data.empty? && res_data.first.is_a?(Hash) && res_data.first.keys.sort.eql?(CONF_OVERVIEW_KEYS)
+        SecretHider.deep_remove_secret(res_data, is_name_value: is_config_overview) unless @option_show_secrets || @option_display.eql?(:data)
         # comma separated list in string format
-        user_asked_fields_list_str=@option_fields
+        user_asked_fields_list_str = @option_fields
         case @option_format
+        when :text
+          display_message(:data,res_data.to_s)
         when :nagios
           Nagios.process(res_data)
         when :ruby
-          display_message(:data,PP.pp(res_data,''))
+          display_message(:data,PP.pp(res_data,+''))
         when :json
           display_message(:data,JSON.generate(res_data))
         when :jsonpp
@@ -165,52 +161,50 @@ module Aspera
           display_message(:data,res_data.to_yaml)
         when :table,:csv
           if !@option_transpose_single && results[:type].eql?(:single_object)
-            results[:type]=:object_list
-            res_data=[res_data]
+            results[:type] = :object_list
+            res_data = [res_data]
           end
           case results[:type]
           when :object_list # goes to table display
             raise "internal error: unexpected type: #{res_data.class}, expecting Array" unless res_data.is_a?(Array)
             # :object_list is an array of hash tables, where key=colum name
             table_rows_hash_val = res_data
-            final_table_columns=nil
+            final_table_columns = nil
             if @option_flat_hash
-              table_rows_hash_val.each do |obj|
-                self.class.flatten_object(obj,results[:option_expand_last])
-              end
+              table_rows_hash_val.map!{|obj|self.class.flattened_object(obj,expand_last: results[:option_expand_last])}
             end
-            final_table_columns=
-            case user_asked_fields_list_str
-            when FIELDS_DEFAULT then result_default_fields(results,table_rows_hash_val)
-            when FIELDS_ALL then     result_all_fields(results,table_rows_hash_val)
-            else
-              if user_asked_fields_list_str.start_with?('+')
-                result_default_fields(results,table_rows_hash_val).push(*user_asked_fields_list_str.gsub(/^\+/,'').split(','))
-              elsif user_asked_fields_list_str.start_with?('-')
-                result_default_fields(results,table_rows_hash_val).reject{|i| user_asked_fields_list_str.gsub(/^-/,'').split(',').include?(i)}
+            final_table_columns =
+              case user_asked_fields_list_str
+              when FIELDS_DEFAULT then result_default_fields(results,table_rows_hash_val)
+              when FIELDS_ALL then     result_all_fields(results,table_rows_hash_val)
               else
-                user_asked_fields_list_str.split(',')
+                if user_asked_fields_list_str.start_with?('+')
+                  result_default_fields(results,table_rows_hash_val).push(*user_asked_fields_list_str.gsub(/^\+/,'').split(','))
+                elsif user_asked_fields_list_str.start_with?('-')
+                  result_default_fields(results,table_rows_hash_val).reject{|i| user_asked_fields_list_str.gsub(/^-/,'').split(',').include?(i)}
+                else
+                  user_asked_fields_list_str.split(',')
+                end
               end
-            end
           when :single_object # goes to table display
             # :single_object is a simple hash table  (can be nested)
             raise "internal error: expecting Hash: got #{res_data.class}: #{res_data}" unless res_data.is_a?(Hash)
-            final_table_columns = results[:columns] || ['key','value']
+            final_table_columns = results[:columns] || %w[key value]
             if @option_flat_hash
-              self.class.flatten_object(res_data,results[:option_expand_last])
+              res_data=self.class.flattened_object(res_data,expand_last: results[:option_expand_last])
               self.class.flatten_name_value_list(res_data)
             end
-            asked_fields=
-            case user_asked_fields_list_str
-            when FIELDS_DEFAULT then results[:fields]||res_data.keys
-            when FIELDS_ALL then     res_data.keys
-            else user_asked_fields_list_str.split(',')
-            end
-            table_rows_hash_val=asked_fields.map { |i| { final_table_columns.first => i, final_table_columns.last => res_data[i] } }
+            asked_fields =
+              case user_asked_fields_list_str
+              when FIELDS_DEFAULT then results[:fields] || res_data.keys
+              when FIELDS_ALL then     res_data.keys
+              else user_asked_fields_list_str.split(',')
+              end
+            table_rows_hash_val = asked_fields.map { |i| { final_table_columns.first => i, final_table_columns.last => res_data[i] } }
           when :value_list # goes to table display
             # :value_list is a simple array of values, name of column provided in the :name
             final_table_columns = [results[:name]]
-            table_rows_hash_val=res_data.map { |i| { results[:name] => i } }
+            table_rows_hash_val = res_data.map { |i| { results[:name] => i } }
           when :empty # no table
             display_message(:info,'empty')
             return
@@ -227,7 +221,7 @@ module Aspera
             return
           when :other_struct # no table
             # :other_struct is any other type of structure
-            display_message(:data,PP.pp(res_data,''))
+            display_message(:data,PP.pp(res_data,+''))
             return
           else
             raise "unknown data type: #{results[:type]}"
@@ -239,18 +233,18 @@ module Aspera
             return
           end
           # convert to string with special function. here table_rows_hash_val is an array of hash
-          table_rows_hash_val=results[:textify].call(table_rows_hash_val) if results.has_key?(:textify)
-          unless @option_select.nil? || (@option_select.respond_to?('empty?') && @option_select.empty?)
+          table_rows_hash_val = results[:textify].call(table_rows_hash_val) if results.has_key?(:textify)
+          unless @option_select.nil? || (@option_select.respond_to?(:empty?) && @option_select.empty?)
             raise CliBadArgument,"expecting hash for select, have #{@option_select.class}: #{@option_select}" unless @option_select.is_a?(Hash)
             @option_select.each{|k,v|table_rows_hash_val.select!{|i|i[k].eql?(v)}}
           end
           # convert data to string, and keep only display fields
-          final_table_rows=table_rows_hash_val.map { |r| final_table_columns.map { |c| r[c].to_s } }
+          final_table_rows = table_rows_hash_val.map { |r| final_table_columns.map { |c| r[c].to_s } }
           # here : final_table_columns : list of column names
           # here: final_table_rows : array of list of value
           case @option_format
           when :table
-            style=@option_table_style.chars
+            style = @option_table_style.chars
             # display the table !
             #display_message(:data,Text::Table.new(
             #head:  final_table_columns,
@@ -259,11 +253,11 @@ module Aspera
             #vertical_boundary:      style[1],
             #boundary_intersection:  style[2]))
             display_message(:data,Terminal::Table.new(
-            headings:  final_table_columns,
-            rows:      final_table_rows,
-            border_x:  style[0],
-            border_y:  style[1],
-            border_i:  style[2]))
+              headings:  final_table_columns,
+              rows:      final_table_rows,
+              border_x:  style[0],
+              border_y:  style[1],
+              border_i:  style[2]))
           when :csv
             display_message(:data,final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
           end

data/lib/aspera/cli/info.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Aspera
   module Cli
     # name of command line tool, also used as foldername where config is stored
@@ -10,8 +11,8 @@ module Aspera
     SRC_URL  = 'https://github.com/IBM/aspera-cli'
     # set this to warn in advance when minimum required ruby version will increase
     # for example currently minimum version is 2.4 in gemspec, but future minimum will be 2.5
-    # set to currenmt minimum if there is no deprecation
+    # set to current minimum if there is no deprecation
     # the actual current minimum required version is in gemspec at required_ruby_version
-    RUBY_FUTURE_MINIMUM_VERSION='2.5'
+    RUBY_FUTURE_MINIMUM_VERSION = '2.5'
   end
 end

data/lib/aspera/cli/listener/line_dump.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/listener'
 require 'json'
 

data/lib/aspera/cli/listener/logger.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/listener'
 require 'aspera/log'
 

data/lib/aspera/cli/listener/progress.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/listener'
 require 'ruby-progressbar'
 
@@ -9,38 +10,38 @@ module Aspera
       class Progress < Fasp::Listener
         def initialize
           super
-          @progress=nil
-          @cumulative=0
+          @progress = nil
+          @cumulative = 0
         end
 
-        BYTE_PER_MEGABIT=(1024*1024)/8
+        BYTE_PER_MEGABIT = (1024 * 1024) / 8
 
         def event_struct(data)
           case data['Type']
           when 'NOTIFICATION'
             if data.has_key?('PreTransferBytes')
-              @progress=ProgressBar.create(
-              format:      '%a %B %p%% %r Mbps %e',
-              rate_scale:  lambda{|rate|rate/BYTE_PER_MEGABIT},
-              title:       'progress',
-              total:       data['PreTransferBytes'].to_i)
+              @progress = ProgressBar.create(
+                format:      '%a %B %p%% %r Mbps %e',
+                rate_scale:  lambda{|rate|rate / BYTE_PER_MEGABIT},
+                title:       'progress',
+                total:       data['PreTransferBytes'].to_i)
             end
           when 'STOP'
             # stop event when one file is completed
-            @cumulative+=data['Size'].to_i
+            @cumulative += data['Size'].to_i
           when 'STATS'
             if @progress.nil?
               puts '.'
             else
-              @progress.progress=data.has_key?('Bytescont') ? @cumulative+data['Bytescont'].to_i : data['TransferBytes'].to_i
+              @progress.progress = data.has_key?('Bytescont') ? @cumulative + data['Bytescont'].to_i : data['TransferBytes'].to_i
             end
           when 'DONE'
             if @progress.nil?
               # terminate progress by going to next line
               puts "\n"
             else
-              @progress.progress=@progress.total
-              @progress=nil
+              @progress.progress = @progress.total
+              @progress = nil
             end
           end
         end

data/lib/aspera/cli/listener/progress_multi.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/listener'
 require 'aspera/fasp/agent_base'
 require 'ruby-progressbar'
@@ -10,20 +11,20 @@ module Aspera
       class ProgressMulti < Fasp::Listener
         def initialize
           super
-          @progress_bar=nil
-          @sessions={}
+          @progress_bar = nil
+          @sessions = {}
         end
 
         def reset
-          @progress_bar=nil
-          @sessions={}
+          @progress_bar = nil
+          @sessions = {}
         end
 
-        BYTE_PER_MEGABIT=1024*1024/8
+        BYTE_PER_MEGABIT = 1024 * 1024 / 8
 
         def update_total
           begin
-            @progress_bar.total=@sessions.values.inject(0){|m,s|m+=s[:job_size].to_i;m;}
+            @progress_bar.total = @sessions.values.inject(0){|m,s|m += s[:job_size].to_i;m;}
           rescue StandardError
             nil
           end
@@ -31,7 +32,7 @@ module Aspera
 
         def update_progress
           begin
-            @progress_bar.progress=@sessions.values.inject(0){|m,s|m+=s[:current].to_i;m;}
+            @progress_bar.progress = @sessions.values.inject(0){|m,s|m += s[:current].to_i;m;}
           rescue StandardError
             nil
           end
@@ -39,41 +40,41 @@ module Aspera
 
         def event_enhanced(data)
           if @progress_bar.nil?
-            @progress_bar=ProgressBar.create(
-            format:      '%t %a %B %p%% %r Mbps %e',
-            rate_scale:  lambda{|rate|rate/BYTE_PER_MEGABIT},
-            title:       '',
-            total:       nil)
+            @progress_bar = ProgressBar.create(
+              format:      '%t %a %B %p%% %r Mbps %e',
+              rate_scale:  lambda{|rate|rate / BYTE_PER_MEGABIT},
+              title:       '',
+              total:       nil)
           end
           if !data.has_key?(Fasp::AgentBase::LISTENER_SESSION_ID_S)
             Log.log.error("Internal error: no #{Fasp::AgentBase::LISTENER_SESSION_ID_S} in event: #{data}")
             return
           end
-          newtitle=@sessions.length < 2 ? '' : "multi=#{@sessions.length}"
-          @progress_bar.title=newtitle unless @progress_bar.title.eql?(newtitle)
-          session=@sessions[data[Fasp::AgentBase::LISTENER_SESSION_ID_S]]||={
+          newtitle = @sessions.length < 2 ? '' : "multi=#{@sessions.length}"
+          @progress_bar.title = newtitle unless @progress_bar.title.eql?(newtitle)
+          session = @sessions[data[Fasp::AgentBase::LISTENER_SESSION_ID_S]] ||= {
             cumulative: 0,
-            job_size: 0,
-            current: 0
+            job_size:   0,
+            current:    0
           }
           case data['type']
           when 'INIT' # connection to ascp (get id)
           when 'SESSION' # session information
           when 'NOTIFICATION' # sent from remote
             if data.has_key?('pre_transfer_bytes')
-              session[:job_size]=data['pre_transfer_bytes']
+              session[:job_size] = data['pre_transfer_bytes']
               update_total
             end
           when 'STATS' # during transfer
             if @progress_bar.total.nil?
               @progress_bar.increment
             else
-              session[:current]=data.has_key?('bytescont') ? session[:cumulative]+data['bytescont'].to_i : data['transfer_bytes'].to_i
+              session[:current] = data.has_key?('bytescont') ? session[:cumulative] + data['bytescont'].to_i : data['transfer_bytes'].to_i
               update_progress
             end
           when 'STOP'
             # stop event when one file is completed
-            session[:cumulative]=session[:cumulative]+data['size'].to_i
+            session[:cumulative] = session[:cumulative] + data['size'].to_i
           when 'DONE' # end of session
             @sessions.delete(data[Fasp::AgentBase::LISTENER_SESSION_ID_S])
             update_progress