aspera-cli 4.8.0 → 4.9.0

data/docs/test_env.conf

@@ -61,7 +61,6 @@ tst_faspex5_web:
   client_secret: your value here
 tst_faspex5:
   url: your value here
-  insecure: your value here
   auth: your value here
   client_id: your value here
   client_secret: your value here
@@ -98,7 +97,6 @@ tst_orch:
   url: your value here
   username: your value here
   password: your value here
-  insecure: your value here
 tst_ats:
   ibm_api_key: your value here
   ats_key: your value here
@@ -108,7 +106,6 @@ tst_bss:
   password: your value here
 tst_ak_preview:
   url: your value here
-  insecure: your value here
   username: your value here
   password: your value here
   mimemagic: your value here
@@ -116,7 +113,6 @@ tst_node_preview:
   url: your value here
   username: your value here
   password: your value here
-  insecure: your value here
 tst_cos:
   apikey: your value here
   crn: your value here
@@ -168,4 +164,4 @@ misc:
   aoc_workspace2: your value here
   http_gw_fqdn_port: your value here
 tst_secrets:
-  st1: your value here
+  eudemo-sedemo: your value here

data/examples/dascli

@@ -3,10 +3,8 @@
 # by default take latest version
 : ${version:=latest}
 imgtag=$image:$version
-# same location as in Dockerfile: generic top folder for apps
-appdir=/usr/src/app
 # same location as in Dockerfile: main config folder for ascli in container
-ascli_home_container=${appdir}/config
+ascli_home_container=/home/cliuser/.aspera/ascli
 # convenience: special argument to install the image
 case "$1" in install) docker pull $imgtag; exit 0; esac
 # set default location for config folder on host if necessary
@@ -20,7 +18,6 @@ exec docker run \
 --rm \
 --tty \
 --interactive \
---env ASCLI_HOME="$ascli_home_container" \
 --volume "$ASCLI_HOME:$ascli_home_container" \
 $imgtag \
 ascli "$@"

data/examples/{transfer.rb → node.rb}

@@ -15,11 +15,12 @@ require 'tmpdir'
 
 tmpdir = ENV['tmp'] || Dir.tmpdir || '.'
 
-DEMO_CONFIG = [
-  'ssh://asperaweb@eudemo.asperademo.com:33001',
-  'https://node_asperaweb@eudemo.asperademo.com:9092',
-  'demoaspera'
-].freeze
+raise 'Usage: PASSWORD=<password> $0 https://<address>:<port> <node user>' unless ARGV.length.eql?(2) && ENV.has_key?('PASSWORD')
+
+# example : https://node_asperaweb@eudemo.asperademo.com:9092
+node_uri = URI.parse(ARGV.shift)
+node_user = ARGV.shift
+node_pass = ENV['PASSWORD']
 
 ##############################################################
 # generic initialisation : configuration of FaspManager
@@ -40,14 +41,14 @@ Aspera::Fasp::Installation.instance.ascp_path = ENV['ascp'] if ENV.has_key?('asc
 # or install:
 #
 
-# get FASP Manager singleton based on above ascp location
-fasp_manager = Aspera::Fasp::AgentDirect.new
+# get Transfer Agent
+transfer_agent = Aspera::Fasp::AgentDirect.new
 
 # Note that it would also be possible to start transfers using other agents
 #require 'aspera/fasp/connect'
-#fasp_manager=Aspera::Fasp::Connect.new
+#transfer_agent=Aspera::Fasp::Connect.new
 #require 'aspera/fasp/node'
-#fasp_manager=Aspera::Fasp::Node.new(Aspera::Rest.new(...))
+#transfer_agent=Aspera::Fasp::Node.new(Aspera::Rest.new(...))
 
 ##############################################################
 # Optional : register an event listener
@@ -60,48 +61,18 @@ class MyListener < Aspera::Fasp::Listener
 end
 
 # register the sample listener to display events
-fasp_manager.add_listener(MyListener.new)
-
-##############################################################
-# first example: download by SSH credentials
-
-# manually build teansfer spec
-transfer_spec = {
-  #'remote_host'     =>'demo.asperasoft.com',
-  'remote_host'      => URI.parse(DEMO_CONFIG[0]).host,
-  'ssh_port'         => URI.parse(DEMO_CONFIG[0]).port,
-  'remote_user'      => URI.parse(DEMO_CONFIG[0]).user,
-  'remote_password'  => DEMO_CONFIG[2],
-  'direction'        => 'receive',
-  'destination_root' => tmpdir,
-  'paths'            => [{'source' => 'aspera-test-dir-tiny/200KB.1'}]
-}
-# start transfer in separate thread
-# method returns as soon as transfer thread is created
-# it des not wait for completion, or even for session startup
-fasp_manager.start_transfer(transfer_spec)
-
-# optional: helper method: wait for completion of transfers
-# here we started a single transfer session (no multisession parameter)
-# get array of status, one for each session (so, a single value array)
-# each status is either :success or "error message"
-transfer_result = fasp_manager.wait_for_transfers_completion
-$stdout.puts(JSON.generate(transfer_result))
-# get list of errors only
-errors = transfer_result.reject{|i|i.eql?(:success)}
-# the transfer was not success, as there is at least one error
-raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
+transfer_agent.add_listener(MyListener.new)
 
 ##############################################################
-# second example: upload with node authorization
+# Upload with node authorization
 
 # create rest client for Node API on a public demo system, using public demo credentials
 node_api = Aspera::Rest.new({
-  base_url: DEMO_CONFIG[1],
+  base_url: node_uri.to_s,
   auth:     {
     type:     :basic,
-    username: URI.parse(DEMO_CONFIG[1]).user,
-    password: DEMO_CONFIG[2]
+    username: node_user,
+    password: node_pass
   }})
 # define sample file(s) and destination folder
 sources = ["#{tmpdir}/sample_file.txt"]
@@ -117,9 +88,9 @@ transfer_spec['paths'] = sources.map{|p|{'source' => p}}
 # set authentication type to "token" (will trigger use of bypass SSH key)
 transfer_spec['authentication'] = 'token'
 # from here : same as example 1
-fasp_manager.start_transfer(transfer_spec)
+transfer_agent.start_transfer(transfer_spec)
 # optional: wait for transfer completion helper function to get events
-transfer_result = fasp_manager.wait_for_transfers_completion
+transfer_result = transfer_agent.wait_for_transfers_completion
 errors = transfer_result.reject{|i|i.eql?(:success)}
 # the transfer was not success, as there is at least one error
 raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?

data/examples/server.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example: transfer a file using one of the provided transfer agents
+# location of ascp can be specified with env var "ascp"
+# temp folder can be specified with env var "tmp"
+require 'aspera/fasp/agent_direct'
+require 'aspera/fasp/listener'
+require 'aspera/fasp/installation'
+require 'aspera/log'
+require 'aspera/rest'
+require 'aspera/rest_errors_aspera'
+require 'json'
+require 'tmpdir'
+
+tmpdir = ENV['tmp'] || Dir.tmpdir || '.'
+
+raise 'Usage: PASSWORD=<password> $0 ssh://<address>:<port> <transfer user>' unless ARGV.length.eql?(2) && ENV.has_key?('PASSWORD')
+
+# example : ssh://asperaweb@eudemo.asperademo.com:33001
+server_uri = URI.parse(ARGV.shift)
+server_user = ARGV.shift
+server_pass = ENV['PASSWORD']
+
+##############################################################
+# generic initialisation : configuration of FaspManager
+
+# set trace level for sample, set to :debug to see complete list of debug information
+Aspera::Log.instance.level = :debug
+
+# register aspera REST call error handlers
+Aspera::RestErrorsAspera.register_handlers
+
+# some required files are generated here (keys, certs)
+Aspera::Fasp::Installation.instance.folder = tmpdir
+# set path to your copy of ascp binary (else, let the system find)
+Aspera::Fasp::Installation.instance.ascp_path = ENV['ascp'] if ENV.has_key?('ascp')
+# another way is to detect installed products and use one of them
+#Aspera::Fasp::Installation.instance.installed_products.each{|p|puts("found: #{p[:name]}")}
+#Aspera::Fasp::Installation.instance.use_ascp_from_product('Aspera Connect')
+# or install:
+#
+
+# get Transfer Agent
+transfer_agent = Aspera::Fasp::AgentDirect.new
+
+# Note that it would also be possible to start transfers using other agents
+#require 'aspera/fasp/connect'
+#transfer_agent=Aspera::Fasp::Connect.new
+#require 'aspera/fasp/node'
+#transfer_agent=Aspera::Fasp::Node.new(Aspera::Rest.new(...))
+
+##############################################################
+# Optional : register an event listener
+
+# example of event listener that displays events on stdout
+class MyListener < Aspera::Fasp::Listener
+  # this is the callback called during transfers, here we only display the received information
+  # but it could be used to get detailed error information, check "type" field is "ERROR"
+  def event_enhanced(data);$stdout.puts(JSON.generate(data));$stdout.flush;end
+end
+
+# register the sample listener to display events
+transfer_agent.add_listener(MyListener.new)
+
+##############################################################
+# first example: download by SSH credentials
+
+# manually build teansfer spec
+transfer_spec = {
+  'remote_host'      => server_uri.host,
+  'ssh_port'         => server_uri.port,
+  'remote_user'      => server_user,
+  'remote_password'  => server_pass,
+  'direction'        => 'receive',
+  'destination_root' => tmpdir,
+  'paths'            => [{'source' => 'aspera-test-dir-tiny/200KB.1'}]
+}
+# start transfer in separate thread
+# method returns as soon as transfer thread is created
+# it des not wait for completion, or even for session startup
+transfer_agent.start_transfer(transfer_spec)
+
+# optional: helper method: wait for completion of transfers
+# here we started a single transfer session (no multisession parameter)
+# get array of status, one for each session (so, a single value array)
+# each status is either :success or "error message"
+transfer_result = transfer_agent.wait_for_transfers_completion
+$stdout.puts(JSON.generate(transfer_result))
+# get list of errors only
+errors = transfer_result.reject{|i|i.eql?(:success)}
+# the transfer was not success, as there is at least one error
+raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?

data/lib/aspera/aoc.rb

@@ -6,6 +6,7 @@ require 'aspera/hash_ext'
 require 'aspera/data_repository'
 require 'aspera/fasp/transfer_spec'
 require 'base64'
+require 'cgi'
 
 Aspera::Oauth.register_token_creator(:aoc_pub_link,lambda{|o|
   o.api.call({
@@ -315,7 +316,8 @@ module Aspera
         end
       else
         # retrieve values from API
-        std_t_spec = node_api.create('files/download_setup',
+        std_t_spec = node_api.create(
+          'files/download_setup',
           {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
         )[:data]['transfer_specs'].first['transfer_spec']
         %w[remote_host remote_user ssh_port fasp_port].each {|i| transfer_spec[i] = std_t_spec[i]}
@@ -453,7 +455,7 @@ module Aspera
     # @param options additional search options
     def lookup_entity_by_name(entity_type,entity_name,options={})
       # returns entities whose name contains value (case insensitive)
-      matching_items = read(entity_type,options.merge({'q' => entity_name}))[:data]
+      matching_items = read(entity_type,options.merge({'q' => CGI.escape(entity_name)}))[:data]
       case matching_items.length
       when 1 then return matching_items.first
       when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{entity_type}: "#{entity_name}"}

data/lib/aspera/ats_api.rb

@@ -18,7 +18,9 @@ module Aspera
 
     private_constant :CLOUD_NAME
 
-    def self.base_url;'https://ats.aspera.io';end
+    class << self
+      def base_url;'https://ats.aspera.io';end
+    end
 
     def initialize
       super({base_url: AtsApi.base_url + '/pub/v1'})

data/lib/aspera/cli/extended_value.rb

@@ -30,6 +30,7 @@ module Aspera
               hasharray.push(entry)
             end
           end
+          Log.log.warn('Titled CSV file without any line') if hasharray.empty?
           return hasharray
         end
       end

data/lib/aspera/cli/formater.rb

@@ -14,7 +14,7 @@ module Aspera
       CSV_RECORD_SEPARATOR = "\n"
       CSV_FIELD_SEPARATOR = ','
       # supported output formats
-      DISPLAY_FORMATS = %i[table ruby json jsonpp yaml csv nagios].freeze
+      DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table csv].freeze
       # user output levels
       DISPLAY_LEVELS = %i[info data error].freeze
       CONF_OVERVIEW_KEYS=%w[config parameter value].freeze
@@ -147,6 +147,8 @@ module Aspera
         # comma separated list in string format
         user_asked_fields_list_str = @option_fields
         case @option_format
+        when :text
+          display_message(:data,res_data.to_s)
         when :nagios
           Nagios.process(res_data)
         when :ruby

data/lib/aspera/cli/info.rb

@@ -11,7 +11,7 @@ module Aspera
     SRC_URL  = 'https://github.com/IBM/aspera-cli'
     # set this to warn in advance when minimum required ruby version will increase
     # for example currently minimum version is 2.4 in gemspec, but future minimum will be 2.5
-    # set to currenmt minimum if there is no deprecation
+    # set to current minimum if there is no deprecation
     # the actual current minimum required version is in gemspec at required_ruby_version
     RUBY_FUTURE_MINIMUM_VERSION = '2.5'
   end

data/lib/aspera/cli/main.rb

@@ -16,6 +16,7 @@ require 'aspera/rest'
 require 'aspera/nagios'
 require 'aspera/colors'
 require 'aspera/secret_hider'
+require 'net/ssh'
 
 module Aspera
   module Cli
@@ -266,12 +267,12 @@ module Aspera
       # early debug for parser
       # Note: does not accept shortcuts
       def early_debug_setup(argv)
-        Log.instance.program_name = PROGRAM_NAME
+        Aspera::Log.instance.program_name = PROGRAM_NAME
         argv.each do |arg|
           case arg
           when '--' then break
-          when /^--log-level=(.*)/ then Log.instance.level = Regexp.last_match(1).to_sym
-          when /^--logger=(.*)/ then Log.instance.logger_type = Regexp.last_match(1).to_sym
+          when /^--log-level=(.*)/ then Aspera::Log.instance.level = Regexp.last_match(1).to_sym
+          when /^--logger=(.*)/ then Aspera::Log.instance.logger_type = Regexp.last_match(1).to_sym
           end
         end
       end
@@ -334,19 +335,21 @@ module Aspera
           @plugin_env[:formater].display_results(command_plugin.execute_action) if execute_command
           # finish
           @plugin_env[:transfer].shutdown
-        rescue CliBadArgument => e;          exception_info = {e: e,t: 'Argument',usage: true}
-        rescue CliNoSuchId => e;             exception_info = {e: e,t: 'Identifier'}
-        rescue CliError => e;                exception_info = {e: e,t: 'Tool',usage: true}
-        rescue Fasp::Error => e;             exception_info = {e: e,t: 'FASP(ascp)'}
-        rescue Aspera::RestCallError => e;   exception_info = {e: e,t: 'Rest'}
-        rescue SocketError => e;             exception_info = {e: e,t: 'Network'}
-        rescue StandardError => e;           exception_info = {e: e,t: 'Other',debug: true}
-        rescue Interrupt => e;               exception_info = {e: e,t: 'Interruption',debug: true}
+        rescue Net::SSH::AuthenticationFailed => e; exception_info = {e: e,t: 'SSH',security: true}
+        rescue CliBadArgument => e;                 exception_info = {e: e,t: 'Argument',usage: true}
+        rescue CliNoSuchId => e;                    exception_info = {e: e,t: 'Identifier'}
+        rescue CliError => e;                       exception_info = {e: e,t: 'Tool',usage: true}
+        rescue Fasp::Error => e;                    exception_info = {e: e,t: 'FASP(ascp)'}
+        rescue Aspera::RestCallError => e;          exception_info = {e: e,t: 'Rest'}
+        rescue SocketError => e;                    exception_info = {e: e,t: 'Network'}
+        rescue StandardError => e;                  exception_info = {e: e,t: 'Other',debug: true}
+        rescue Interrupt => e;                      exception_info = {e: e,t: 'Interruption',debug: true}
         end
         # cleanup file list files
         TempFileManager.instance.cleanup
         # 1- processing of error condition
         unless exception_info.nil?
+          Log.log.warn(exception_info[:e].message) if Aspera::Log.instance.logger_type.eql?(:syslog) && exception_info[:security]
           @plugin_env[:formater].display_message(:error,"#{ERROR_FLASH} #{exception_info[:t]}: #{exception_info[:e].message}")
           @plugin_env[:formater].display_message(:error,'Use option -h to get help.') if exception_info[:usage]
           if exception_info[:e].is_a?(Fasp::Error) && exception_info[:e].message.eql?('Remote host is not who we expected')

data/lib/aspera/cli/manager.rb

@@ -56,7 +56,7 @@ module Aspera
 
       class << self
         def enum_to_bool(enum)
-          raise "Value not valid for boolean: #{enum}/#{enum.class}" unless BOOLEAN_VALUES.include?(enum)
+          raise "Value not valid for boolean: [#{enum}]/#{enum.class}" unless BOOLEAN_VALUES.include?(enum)
           return TRUE_VALUES.include?(enum)
         end
 
@@ -210,7 +210,7 @@ module Aspera
         end
         value = ExtendedValue.instance.evaluate(value)
         value = Manager.enum_to_bool(value) if @declared_options[option_symbol][:values].eql?(BOOLEAN_VALUES)
-        Log.log.debug("set #{option_symbol}=#{value} (#{@declared_options[option_symbol][:type]}) : #{where}")
+        Log.log.debug("(#{@declared_options[option_symbol][:type]}/#{where}) set #{option_symbol}=#{value}")
         case @declared_options[option_symbol][:type]
         when :accessor
           @declared_options[option_symbol][:accessor].value = value
@@ -235,10 +235,10 @@ module Aspera
           else
             raise 'unknown type'
           end
-          Log.log.debug("get #{option_symbol} (#{@declared_options[option_symbol][:type]}) : #{result}")
+          Log.log.debug("(#{@declared_options[option_symbol][:type]}) get #{option_symbol}=#{result}")
         end
         # do not fail for manual generation if option mandatory but not set
-        result = '' if result.nil? && !@fail_on_missing_mandatory
+        result = '' if result.nil? && is_type.eql?(:mandatory) && !@fail_on_missing_mandatory
         #Log.log.debug("interactive=#{@ask_missing_mandatory}")
         if result.nil?
           if !@ask_missing_mandatory

data/lib/aspera/cli/plugin.rb

@@ -4,9 +4,9 @@ module Aspera
   module Cli
     # base class for plugins modules
     class Plugin
-      # operation without id
+      # operations without id
       GLOBAL_OPS = %i[create list].freeze
-      # operation on specific instance
+      # operations with id
       INSTANCE_OPS = %i[modify delete show].freeze
       ALL_OPS = [GLOBAL_OPS,INSTANCE_OPS].flatten.freeze
       # max number of items for list command
@@ -34,6 +34,8 @@ module Aspera
         options.add_opt_simple(:value,'extended value for create, update, list filter')
         options.add_opt_simple(:property,'name of property to set')
         options.add_opt_simple(:id,"resource identifier (#{INSTANCE_OPS.join(',')})")
+        options.add_opt_boolean(:bulk,'Bulk operation (only some)')
+        options.set_option(:bulk,:no)
         options.parse_options!
         @@options_created = true # rubocop:disable Style/ClassVars
       end
@@ -46,8 +48,41 @@ module Aspera
       end
 
       # TODO
-      def get_next_id_command(instance_ops: INSTANCE_OPS,global_ops: GLOBAL_OPS)
-        return get_next_argument('command',expected: command_list)
+      #def get_next_id_command(instance_ops: INSTANCE_OPS,global_ops: GLOBAL_OPS)
+      #  return get_next_argument('command',expected: command_list)
+      #end
+
+      # For create and delete operations: execute one actin or multiple if bulk is yes
+      # @param params either single id or hash, or array for bulk
+      # @param success_msg deleted or created
+      def do_bulk_operation(single_or_array,success_msg,id_result: 'id',fields: :default)
+        raise 'programming error: missing block' unless block_given?
+        params = options.get_option(:bulk) ? single_or_array : [single_or_array]
+        raise 'expecting Array for bulk operation' unless params.is_a?(Array)
+        Log.log.warn('Empty list given for bulk operation') if params.empty?
+        Log.dump(:bulk_create,params)
+        result_list = []
+        params.each do |param|
+          # init for delete
+          result = {id_result => param}
+          begin
+            # execute custom code
+            res = yield(param)
+            # if block returns a hash, let's use this (create)
+            result = res if param.is_a?(Hash)
+            result['status'] = success_msg
+          rescue StandardError => e
+            result['status'] = e.to_s
+          end
+          result_list.push(result)
+        end
+        display_fields = [id_result,'status']
+        if options.get_option(:bulk)
+          return {type: :object_list,data: result_list,fields: display_fields}
+        else
+          display_fields = fields unless fields.eql?(:default)
+          return {type: :single_object,data: result_list.first,fields: display_fields}
+        end
       end
 
       # @param command [Symbol] command to execute: create show list modify delete
@@ -56,6 +91,7 @@ module Aspera
       # @param display_fields [Array] fields to display by default
       # @param id_default [String] default identifier to use for existing entity commands (show, modify)
       # @param item_list_key [String] result is in a subkey of the json
+      # @return result suitable for CLI result
       def entity_command(command,rest_api,res_class_path,display_fields: nil,id_default: nil,item_list_key: false)
         if INSTANCE_OPS.include?(command)
           begin
@@ -76,7 +112,15 @@ module Aspera
         end
         case command
         when :create
-          return {type: :single_object, data: rest_api.create(res_class_path,parameters)[:data], fields: display_fields}
+          return do_bulk_operation(parameters,'created',fields: display_fields) do |params|
+            raise 'expecting Hash' unless params.is_a?(Hash)
+            rest_api.create(res_class_path,params)[:data]
+          end
+        when :delete
+          return do_bulk_operation(one_res_id,'deleted') do |one_id|
+            rest_api.delete("#{res_class_path}/#{one_id}")
+            {'id' => one_id}
+          end
         when :show
           return {type: :single_object, data: rest_api.read(one_res_path)[:data], fields: display_fields}
         when :list
@@ -102,9 +146,6 @@ module Aspera
           parameters = {property => parameters} unless property.nil?
           rest_api.update(one_res_path,parameters)
           return Main.result_status('modified')
-        when :delete
-          rest_api.delete(one_res_path)
-          return Main.result_status('deleted')
         else
           raise "unknown action: #{command}"
         end
@@ -117,7 +158,7 @@ module Aspera
         return entity_command(command,rest_api,res_class_path,**opts)
       end
 
-      # shortcuts for plugin environment
+      # shortcuts helpers for plugin environment
       def options; return @agents[:options];end
 
       def transfer; return @agents[:transfer];end