aspera-cli 4.6.0 → 4.7.0

data/lib/aspera/cli/formater.rb

@@ -1,7 +1,5 @@
-#require 'text-table'
+# frozen_string_literal: true
 require 'terminal-table'
-
-#require 'fileutils'
 require 'yaml'
 require 'pp'
 
@@ -12,41 +10,96 @@ module Aspera
       FIELDS_ALL='ALL'
       FIELDS_DEFAULT='DEF'
       # supported output formats
-      DISPLAY_FORMATS=[:table,:ruby,:json,:jsonpp,:yaml,:csv,:nagios]
+      DISPLAY_FORMATS=%i[table ruby json jsonpp yaml csv nagios]
       # user output levels
       DISPLAY_LEVELS=[:info,:data,:error]
       CSV_RECORD_SEPARATOR="\n"
-      CSV_FIELD_SEPARATOR=","
+      CSV_FIELD_SEPARATOR=','
+      HIDDEN_PASSWORD='🔑'
+      SECRET_KEYWORDS=%w[password secret private_key]
+
+      private_constant :FIELDS_ALL,:FIELDS_DEFAULT,:DISPLAY_FORMATS,:DISPLAY_LEVELS,:CSV_RECORD_SEPARATOR,:CSV_FIELD_SEPARATOR,:HIDDEN_PASSWORD
 
-      private_constant :FIELDS_ALL,:FIELDS_DEFAULT,:DISPLAY_FORMATS,:DISPLAY_LEVELS,:CSV_RECORD_SEPARATOR,:CSV_FIELD_SEPARATOR
-      attr_accessor :option_flat_hash,:option_transpose_single
+      class<<self
+        # @param source [Hash] hash to modify
+        # @param keep_last [bool]
+        def flatten_object(source,keep_last)
+          newval={}
+          flatten_sub_hash_rec(source,keep_last,'',newval)
+          source.clear
+          source.merge!(newval)
+        end
 
+        # recursive function to modify a hash
+        # @param source [Hash] to be modified
+        # @param keep_last [bool] truer if last level is not
+        # @param prefix [String] true if last level is not
+        # @param dest [Hash] new hash flattened
+        def flatten_sub_hash_rec(source,keep_last,prefix,dest)
+          #is_simple_hash=source.is_a?(Hash) and source.values.inject(true){|m,v| xxx=!v.respond_to?(:each) and m;puts("->#{xxx}>#{v.respond_to?(:each)} #{v}-");xxx}
+          is_simple_hash=false
+          Log.log.debug("(#{keep_last})[#{is_simple_hash}] -#{source.values}- \n-#{source}-")
+          return source if keep_last && is_simple_hash
+          source.each do |k,v|
+            if v.is_a?(Hash) && (!keep_last || !is_simple_hash)
+              flatten_sub_hash_rec(v,keep_last,prefix+k.to_s+'.',dest)
+            else
+              dest[prefix+k.to_s]=v
+            end
+          end
+          return nil
+        end
+
+        # special for Aspera on Cloud display node
+        # {"param" => [{"name"=>"foo","value"=>"bar"}]} will be expanded to {"param.foo" : "bar"}
+        def flatten_name_value_list(hash)
+          hash.keys.each do |k|
+            v=hash[k]
+            next unless v.is_a?(Array) && v.map(&:class).uniq.eql?([Hash]) && v.map(&:keys).flatten.sort.uniq.eql?(['name', 'value'])
+            v.each do |pair|
+              hash["#{k}.#{pair['name']}"]=pair['value']
+            end
+            hash.delete(k)
+          end
+        end
+      end
+
+      attr_accessor :option_flat_hash,:option_transpose_single,:option_format,:option_display,:option_fields,:option_table_style,:option_select,:option_show_secrets
+
+      # adds options but does not parse
       def initialize(opt_mgr)
+        @option_format=:table
+        @option_display=:info
+        @option_fields=FIELDS_DEFAULT
+        @option_select=nil
+        @option_table_style=':.:'
         @option_flat_hash=true
         @option_transpose_single=true
-        @opt_mgr=opt_mgr
-        @opt_mgr.set_obj_attr(:flat_hash,self,:option_flat_hash)
-        @opt_mgr.set_obj_attr(:transpose_single,self,:option_transpose_single)
-        @opt_mgr.add_opt_list(:format,DISPLAY_FORMATS,"output format")
-        @opt_mgr.add_opt_list(:display,DISPLAY_LEVELS,"output only some information")
-        @opt_mgr.add_opt_simple(:fields,"comma separated list of fields, or #{FIELDS_ALL}, or #{FIELDS_DEFAULT}")
-        @opt_mgr.add_opt_simple(:select,"select only some items in lists, extended value: hash (column, value)")
-        @opt_mgr.add_opt_simple(:table_style,"table display style")
-        @opt_mgr.add_opt_boolean(:flat_hash,"display hash values as additional keys")
-        @opt_mgr.add_opt_boolean(:transpose_single,"single object fields output vertically")
-        @opt_mgr.set_option(:format,:table)
-        @opt_mgr.set_option(:display,:info)
-        @opt_mgr.set_option(:fields,FIELDS_DEFAULT)
-        @opt_mgr.set_option(:table_style,':.:')
+        @option_show_secrets=true
+        opt_mgr.set_obj_attr(:format,self,:option_format)
+        opt_mgr.set_obj_attr(:display,self,:option_display)
+        opt_mgr.set_obj_attr(:fields,self,:option_fields)
+        opt_mgr.set_obj_attr(:select,self,:option_select)
+        opt_mgr.set_obj_attr(:table_style,self,:option_table_style)
+        opt_mgr.set_obj_attr(:flat_hash,self,:option_flat_hash)
+        opt_mgr.set_obj_attr(:transpose_single,self,:option_transpose_single)
+        opt_mgr.set_obj_attr(:show_secrets,self,:option_show_secrets)
+        opt_mgr.add_opt_list(:format,DISPLAY_FORMATS,'output format')
+        opt_mgr.add_opt_list(:display,DISPLAY_LEVELS,'output only some information')
+        opt_mgr.add_opt_simple(:fields,"comma separated list of fields, or #{FIELDS_ALL}, or #{FIELDS_DEFAULT}")
+        opt_mgr.add_opt_simple(:select,'select only some items in lists, extended value: hash (column, value)')
+        opt_mgr.add_opt_simple(:table_style,'table display style')
+        opt_mgr.add_opt_boolean(:flat_hash,'display hash values as additional keys')
+        opt_mgr.add_opt_boolean(:transpose_single,'single object fields output vertically')
+        opt_mgr.add_opt_boolean(:show_secrets,'show secrets on command output')
       end
 
       # main output method
       def display_message(message_level,message)
-        display_level=@opt_mgr.get_option(:display,:mandatory)
         case message_level
-        when :info; STDOUT.puts(message) if display_level.eql?(:info)
-        when :data; STDOUT.puts(message) unless display_level.eql?(:error)
-        when :error; STDERR.puts(message)
+        when :info then $stdout.puts(message) if @option_display.eql?(:info)
+        when :data then $stdout.puts(message) unless @option_display.eql?(:error)
+        when :error then $stderr.puts(message)
         else raise "wrong message_level:#{message_level}"
         end
       end
@@ -55,77 +108,51 @@ module Aspera
         display_message(:info,status)
       end
 
-      # @param source [Hash] hash to modify
-      # @param keep_last [bool]
-      def self.flatten_object(source,keep_last)
-        newval={}
-        flatten_sub_hash_rec(source,keep_last,'',newval)
-        source.clear
-        source.merge!(newval)
-      end
-
-      # recursive function to modify a hash
-      # @param source [Hash] to be modified
-      # @param keep_last [bool] truer if last level is not
-      # @param prefix [String] true if last level is not
-      # @param dest [Hash] new hash flattened
-      def self.flatten_sub_hash_rec(source,keep_last,prefix,dest)
-        #is_simple_hash=source.is_a?(Hash) and source.values.inject(true){|m,v| xxx=!v.respond_to?(:each) and m;puts("->#{xxx}>#{v.respond_to?(:each)} #{v}-");xxx}
-        is_simple_hash=false
-        Log.log.debug("(#{keep_last})[#{is_simple_hash}] -#{source.values}- \n-#{source}-")
-        return source if keep_last and is_simple_hash
-        source.each do |k,v|
-          if v.is_a?(Hash) and ( !keep_last or !is_simple_hash )
-            flatten_sub_hash_rec(v,keep_last,prefix+k.to_s+'.',dest)
-          else
-            dest[prefix+k.to_s]=v
+      def result_default_fields(results,table_rows_hash_val)
+        unless results[:fields].nil?
+          raise "internal error: [fields] must be Array, not #{results[:fields].class}" unless results[:fields].is_a?(Array)
+          if results[:fields].first.eql?(:all_but) && !table_rows_hash_val.empty?
+            filter=results[:fields][1..-1]
+            return table_rows_hash_val.first.keys.reject{|i|filter.include?(i)}
           end
+          return results[:fields]
         end
-        return nil
+        return ['empty'] if table_rows_hash_val.empty?
+        return table_rows_hash_val.first.keys
       end
 
-      # special for Aspera on Cloud display node
-      # {"param" => [{"name"=>"foo","value"=>"bar"}]} will be expanded to {"param.foo" : "bar"}
-      def self.flatten_name_value_list(hash)
-        hash.keys.each do |k|
-          v=hash[k]
-          if v.is_a?(Array) and v.map{|i|i.class}.uniq.eql?([Hash]) and v.map{|i|i.keys}.flatten.sort.uniq.eql?(["name", "value"])
-            v.each do |pair|
-              hash["#{k}.#{pair["name"]}"]=pair["value"]
-            end
-            hash.delete(k)
-          end
-        end
+      def result_all_fields(_results,table_rows_hash_val)
+        raise 'internal error: must be array' unless table_rows_hash_val.is_a?(Array)
+        # get the list of all column names used in all lines, not just frst one, as all lines may have different columns
+        return table_rows_hash_val.each_with_object({}){|v,m|v.keys.each{|c|m[c]=true};}.keys
       end
 
-      def result_default_fields(results,table_rows_hash_val)
-        if results.has_key?(:fields) and !results[:fields].nil?
-          final_table_columns=results[:fields]
-        else
-          if !table_rows_hash_val.empty?
-            final_table_columns=table_rows_hash_val.first.keys
-          else
-            final_table_columns=['empty']
+      def deep_remove_secret(obj)
+        return if @option_show_secrets
+        case obj
+        when Array then obj.each{|i|deep_remove_secret(i)}
+        when Hash
+          obj.keys.each do |k|
+            next if %w[show_secrets log_secrets].any?{|kw|kw.eql?(k)}
+            if k.is_a?(String) && SECRET_KEYWORDS.any?{|kw|k.include?(kw)}
+              obj[k]=HIDDEN_PASSWORD
+            elsif obj[k].is_a?(Hash)
+              deep_remove_secret(obj[k])
+            end
           end
         end
       end
 
-      def result_all_fields(results,table_rows_hash_val)
-        raise "internal error: must be array" unless table_rows_hash_val.is_a?(Array)
-        # get the list of all column names used in all lines, not just frst one, as all lines may have different columns
-        return table_rows_hash_val.inject({}){|m,v|v.keys.each{|c|m[c]=true};m}.keys
-      end
-
       # this method displays the results, especially the table format
       def display_results(results)
         raise "INTERNAL ERROR, result must be Hash (got: #{results.class}: #{results})" unless results.is_a?(Hash)
-        raise "INTERNAL ERROR, result must have type" unless results.has_key?(:type)
-        raise "INTERNAL ERROR, result must have data" unless results.has_key?(:data) or [:empty,:nothing].include?(results[:type])
+        raise 'INTERNAL ERROR, result must have type' unless results.has_key?(:type)
+        raise 'INTERNAL ERROR, result must have data' unless results.has_key?(:data) || [:empty,:nothing].include?(results[:type])
         res_data=results[:data]
+        deep_remove_secret(res_data)
         # comma separated list in string format
-        user_asked_fields_list_str=@opt_mgr.get_option(:fields,:mandatory)
-        display_format=@opt_mgr.get_option(:format,:mandatory)
-        case display_format
+        user_asked_fields_list_str=@option_fields
+        case @option_format
         when :nagios
           Nagios.process(res_data)
         when :ruby
@@ -137,7 +164,7 @@ module Aspera
         when :yaml
           display_message(:data,res_data.to_yaml)
         when :table,:csv
-          if !@option_transpose_single and results[:type].eql?(:single_object)
+          if !@option_transpose_single && results[:type].eql?(:single_object)
             results[:type]=:object_list
             res_data=[res_data]
           end
@@ -152,14 +179,15 @@ module Aspera
                 self.class.flatten_object(obj,results[:option_expand_last])
               end
             end
-            final_table_columns=case user_asked_fields_list_str
-            when FIELDS_DEFAULT; result_default_fields(results,table_rows_hash_val)
-            when FIELDS_ALL;     result_all_fields(results,table_rows_hash_val)
+            final_table_columns=
+            case user_asked_fields_list_str
+            when FIELDS_DEFAULT then result_default_fields(results,table_rows_hash_val)
+            when FIELDS_ALL then     result_all_fields(results,table_rows_hash_val)
             else
               if user_asked_fields_list_str.start_with?('+')
                 result_default_fields(results,table_rows_hash_val).push(*user_asked_fields_list_str.gsub(/^\+/,'').split(','))
               elsif user_asked_fields_list_str.start_with?('-')
-                result_default_fields(results,table_rows_hash_val).select{|i| ! user_asked_fields_list_str.gsub(/^\-/,'').split(',').include?(i)}
+                result_default_fields(results,table_rows_hash_val).reject{|i| user_asked_fields_list_str.gsub(/^-/,'').split(',').include?(i)}
               else
                 user_asked_fields_list_str.split(',')
               end
@@ -172,13 +200,14 @@ module Aspera
               self.class.flatten_object(res_data,results[:option_expand_last])
               self.class.flatten_name_value_list(res_data)
             end
-            asked_fields=case user_asked_fields_list_str
-            when FIELDS_DEFAULT; results[:fields]||res_data.keys
-            when FIELDS_ALL;     res_data.keys
-            else                 user_asked_fields_list_str.split(',')
+            asked_fields=
+            case user_asked_fields_list_str
+            when FIELDS_DEFAULT then results[:fields]||res_data.keys
+            when FIELDS_ALL then     res_data.keys
+            else user_asked_fields_list_str.split(',')
             end
             table_rows_hash_val=asked_fields.map { |i| { final_table_columns.first => i, final_table_columns.last => res_data[i] } }
-          when :value_list  # goes to table display
+          when :value_list # goes to table display
             # :value_list is a simple array of values, name of column provided in the :name
             final_table_columns = [results[:name]]
             table_rows_hash_val=res_data.map { |i| { results[:name] => i } }
@@ -186,7 +215,7 @@ module Aspera
             display_message(:info,'empty')
             return
           when :nothing # no result expected
-            Log.log.debug("no result expected")
+            Log.log.debug('no result expected')
             return
           when :status # no table
             # :status displays a simple message
@@ -204,39 +233,37 @@ module Aspera
             raise "unknown data type: #{results[:type]}"
           end
           # here we expect: table_rows_hash_val and final_table_columns
-          raise "no field specified" if final_table_columns.nil?
+          raise 'no field specified' if final_table_columns.nil?
           if table_rows_hash_val.empty?
-            display_message(:info,'empty'.gray) unless display_format.eql?(:csv)
+            display_message(:info,'empty'.gray) unless @option_format.eql?(:csv)
             return
           end
           # convert to string with special function. here table_rows_hash_val is an array of hash
           table_rows_hash_val=results[:textify].call(table_rows_hash_val) if results.has_key?(:textify)
-          filter=@opt_mgr.get_option(:select,:optional)
-          unless filter.nil?
-            raise CliBadArgument,"expecting hash for select" unless filter.is_a?(Hash)
-            filter.each{|k,v|table_rows_hash_val.select!{|i|i[k].eql?(v)}}
+          unless @option_select.nil? || (@option_select.respond_to?('empty?') && @option_select.empty?)
+            raise CliBadArgument,"expecting hash for select, have #{@option_select.class}: #{@option_select}" unless @option_select.is_a?(Hash)
+            @option_select.each{|k,v|table_rows_hash_val.select!{|i|i[k].eql?(v)}}
           end
-
           # convert data to string, and keep only display fields
           final_table_rows=table_rows_hash_val.map { |r| final_table_columns.map { |c| r[c].to_s } }
           # here : final_table_columns : list of column names
           # here: final_table_rows : array of list of value
-          case display_format
+          case @option_format
           when :table
-            style=@opt_mgr.get_option(:table_style,:mandatory).split('')
+            style=@option_table_style.chars
             # display the table !
             #display_message(:data,Text::Table.new(
-            #:head => final_table_columns,
-            #:rows => final_table_rows,
-            #:horizontal_boundary   => style[0],
-            #:vertical_boundary     => style[1],
-            #:boundary_intersection => style[2]))
+            #head:  final_table_columns,
+            #rows:  final_table_rows,
+            #horizontal_boundary:    style[0],
+            #vertical_boundary:      style[1],
+            #boundary_intersection:  style[2]))
             display_message(:data,Terminal::Table.new(
-            :headings => final_table_columns,
-            :rows     => final_table_rows,
-            :border_x => style[0],
-            :border_y => style[1],
-            :border_i => style[2]))
+            headings:  final_table_columns,
+            rows:      final_table_rows,
+            border_x:  style[0],
+            border_y:  style[1],
+            border_i:  style[2]))
           when :csv
             display_message(:data,final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
           end

data/lib/aspera/cli/info.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+module Aspera
+  module Cli
+    # name of command line tool, also used as foldername where config is stored
+    PROGRAM_NAME = 'ascli'
+    # name of the containing gem, same as in <gem name>.gemspec
+    GEM_NAME = 'aspera-cli'
+    DOC_URL  = "https://www.rubydoc.info/gems/#{GEM_NAME}"
+    GEM_URL  = "https://rubygems.org/gems/#{GEM_NAME}"
+    SRC_URL  = 'https://github.com/IBM/aspera-cli'
+    # set this to warn in advance when minimum required ruby version will increase
+    # for example currently minimum version is 2.4 in gemspec, but future minimum will be 2.5
+    # set to currenmt minimum if there is no deprecation
+    # the actual current minimum required version is in gemspec at required_ruby_version
+    RUBY_FUTURE_MINIMUM_VERSION='2.5'
+  end
+end

data/lib/aspera/cli/listener/line_dump.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'aspera/fasp/listener'
 require 'json'
 
@@ -8,8 +9,8 @@ module Aspera
       # FASP event listener display management events as JSON
       class LineDump < Fasp::Listener
         def event_enhanced(data)
-          STDOUT.puts(JSON.generate(data))
-          STDOUT.flush
+          $stdout.puts(JSON.generate(data))
+          $stdout.flush
         end
       end
     end

data/lib/aspera/cli/listener/logger.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'aspera/fasp/listener'
 require 'aspera/log'
 
@@ -8,7 +9,7 @@ module Aspera
       class Logger < Fasp::Listener
         def event_struct(data)
           Log.log.debug(data.to_s)
-          Log.log.error("#{data['Description']}") if data['Type'].eql?('FILEERROR')
+          Log.log.error((data['Description']).to_s) if data['Type'].eql?('FILEERROR')
         end
 
         def event_enhanced(data)

data/lib/aspera/cli/listener/progress.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'aspera/fasp/listener'
 require 'ruby-progressbar'
 
@@ -7,42 +8,39 @@ module Aspera
       # a listener to FASP event that displays a progress bar
       class Progress < Fasp::Listener
         def initialize
+          super
           @progress=nil
           @cumulative=0
         end
 
-        BYTE_PER_MEGABIT=1024*1024/8
+        BYTE_PER_MEGABIT=(1024*1024)/8
 
         def event_struct(data)
           case data['Type']
           when 'NOTIFICATION'
-            if data.has_key?('PreTransferBytes') then
+            if data.has_key?('PreTransferBytes')
               @progress=ProgressBar.create(
-              :format     => '%a %B %p%% %r Mbps %e',
-              :rate_scale => lambda{|rate|rate/BYTE_PER_MEGABIT},
-              :title      => 'progress',
-              :total      => data['PreTransferBytes'].to_i)
+              format:      '%a %B %p%% %r Mbps %e',
+              rate_scale:  lambda{|rate|rate/BYTE_PER_MEGABIT},
+              title:       'progress',
+              total:       data['PreTransferBytes'].to_i)
             end
           when 'STOP'
             # stop event when one file is completed
-            @cumulative=@cumulative+data['Size'].to_i
+            @cumulative+=data['Size'].to_i
           when 'STATS'
-            if !@progress.nil? then
-              if data.has_key?('Bytescont')
-                @progress.progress=@cumulative+data['Bytescont'].to_i
-              else
-                @progress.progress=data['TransferBytes'].to_i
-              end
-            else
+            if @progress.nil?
               puts '.'
+            else
+              @progress.progress=data.has_key?('Bytescont') ? @cumulative+data['Bytescont'].to_i : data['TransferBytes'].to_i
             end
           when 'DONE'
-            if !@progress.nil? then
-              @progress.progress=@progress.total
-              @progress=nil
-            else
+            if @progress.nil?
               # terminate progress by going to next line
               puts "\n"
+            else
+              @progress.progress=@progress.total
+              @progress=nil
             end
           end
         end

data/lib/aspera/cli/listener/progress_multi.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'aspera/fasp/listener'
 require 'aspera/fasp/agent_base'
 require 'ruby-progressbar'
@@ -8,6 +9,7 @@ module Aspera
       # a listener to FASP event that displays a progress bar
       class ProgressMulti < Fasp::Listener
         def initialize
+          super
           @progress_bar=nil
           @sessions={}
         end
@@ -22,7 +24,7 @@ module Aspera
         def update_total
           begin
             @progress_bar.total=@sessions.values.inject(0){|m,s|m+=s[:job_size].to_i;m;}
-          rescue
+          rescue StandardError
             nil
           end
         end
@@ -30,7 +32,7 @@ module Aspera
         def update_progress
           begin
             @progress_bar.progress=@sessions.values.inject(0){|m,s|m+=s[:current].to_i;m;}
-          rescue
+          rescue StandardError
             nil
           end
         end
@@ -38,10 +40,10 @@ module Aspera
         def event_enhanced(data)
           if @progress_bar.nil?
             @progress_bar=ProgressBar.create(
-            :format     => '%t %a %B %p%% %r Mbps %e',
-            :rate_scale => lambda{|rate|rate/BYTE_PER_MEGABIT},
-            :title      => '',
-            :total      => nil)
+            format:      '%t %a %B %p%% %r Mbps %e',
+            rate_scale:  lambda{|rate|rate/BYTE_PER_MEGABIT},
+            title:       '',
+            total:       nil)
           end
           if !data.has_key?(Fasp::AgentBase::LISTENER_SESSION_ID_S)
             Log.log.error("Internal error: no #{Fasp::AgentBase::LISTENER_SESSION_ID_S} in event: #{data}")
@@ -58,21 +60,16 @@ module Aspera
           when 'INIT' # connection to ascp (get id)
           when 'SESSION' # session information
           when 'NOTIFICATION' # sent from remote
-            if data.has_key?('pre_transfer_bytes') then
+            if data.has_key?('pre_transfer_bytes')
               session[:job_size]=data['pre_transfer_bytes']
               update_total
             end
           when 'STATS' # during transfer
-            if !@progress_bar.total.nil? then
-              if data.has_key?('bytescont')
-                session[:current]=session[:cumulative]+data['bytescont'].to_i
-                update_progress
-              else
-                session[:current]=data['transfer_bytes'].to_i
-                update_progress
-              end
-            else
+            if @progress_bar.total.nil?
               @progress_bar.increment
+            else
+              session[:current]=data.has_key?('bytescont') ? session[:cumulative]+data['bytescont'].to_i : data['transfer_bytes'].to_i
+              update_progress
             end
           when 'STOP'
             # stop event when one file is completed