aspera-cli 4.14.0 → 4.15.0

data/lib/aspera/cli/plugins/console.rb

@@ -7,6 +7,46 @@ module Aspera
   module Cli
     module Plugins
       class Console < Aspera::Cli::BasicAuthPlugin
+        STANDARD_PATH = '/aspera/console'
+        class << self
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+
+            urls.each do |base_url|
+              next unless base_url.start_with?('https://')
+              api = Rest.new(base_url: base_url, redirect_max: 2)
+              test_endpoint = 'login'
+              test_page = api.call({operation: 'GET', subpath: test_endpoint, url_params: {local: true}})
+              next unless test_page[:http].body.include?('Aspera Console')
+              version = 'unknown'
+              if (m = test_page[:http].body.match(/\(v([1-9]\..*)\)/))
+                version = m[1]
+              end
+              url = test_page[:http].uri.to_s
+              return {
+                version: version,
+                url:     url[0..url.index(test_endpoint) - 2]
+              }
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
+            end
+            return nil
+          end
+
+          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+            options = object.options
+            return {
+              preset_value: {
+                url:      options.get_option(:url, mandatory: true),
+                username: options.get_option(:username, mandatory: true),
+                password: options.get_option(:password, mandatory: true)
+              },
+              test_args:    'transfer list'
+            }
+          end
+        end
         DEFAULT_FILTER_AGE_SECONDS = 3 * 3600
         private_constant :DEFAULT_FILTER_AGE_SECONDS
         def initialize(env)

data/lib/aspera/cli/plugins/cos.rb

@@ -10,7 +10,6 @@ module Aspera
       class Cos < Aspera::Cli::Plugin
         def initialize(env)
           super(env)
-          @service_creds = nil
           options.declare(:bucket, 'Bucket name')
           options.declare(:endpoint, 'Storage endpoint url')
           options.declare(:apikey, 'Storage API key')
@@ -31,19 +30,19 @@ module Aspera
             # get service credentials, Hash, e.g. @json:@file:...
             service_credentials = options.get_option(:service_credentials)
             storage_endpoint = options.get_option(:endpoint)
-            raise CliBadArgument, 'one of: endpoint or service_credentials is required' if service_credentials.nil? && storage_endpoint.nil?
-            raise CliBadArgument, 'endpoint and service_credentials are mutually exclusive' unless service_credentials.nil? || storage_endpoint.nil?
+            raise Cli::BadArgument, 'one of: endpoint or service_credentials is required' if service_credentials.nil? && storage_endpoint.nil?
+            raise Cli::BadArgument, 'endpoint and service_credentials are mutually exclusive' unless service_credentials.nil? || storage_endpoint.nil?
             if service_credentials.nil?
               service_api_key = options.get_option(:apikey, mandatory: true)
               instance_id = options.get_option(:crn, mandatory: true)
             else
-              params = CosNode.parameters_from_svc_creds(service_credentials, options.get_option(:region, mandatory: true))
+              params = CosNode.parameters_from_svc_credentials(service_credentials, options.get_option(:region, mandatory: true))
               storage_endpoint = params[:storage_endpoint]
               service_api_key = params[:service_api_key]
               instance_id = params[:instance_id]
             end
             api_node = CosNode.new(bucket_name, storage_endpoint, instance_id, service_api_key, options.get_option(:identity, mandatory: true))
-            node_plugin = Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node))
+            node_plugin = Node.new(@agents, api: api_node)
             command = options.get_next_command(Node::COMMANDS_COS)
             return node_plugin.execute_action(command)
           end

data/lib/aspera/cli/plugins/faspex.rb

@@ -1,14 +1,15 @@
 # frozen_string_literal: true
 
+# cspell:ignore passcode xrds workgroups dmembership wmembership
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/cli/plugins/node'
 require 'aspera/cli/plugins/config'
 require 'aspera/cli/extended_value'
 require 'aspera/cli/transfer_agent'
-require 'aspera/persistency_action_once'
-require 'aspera/open_application'
 require 'aspera/fasp/uri'
 require 'aspera/fasp/transfer_spec'
+require 'aspera/persistency_action_once'
+require 'aspera/open_application'
 require 'aspera/nagios'
 require 'aspera/id_generator'
 require 'xmlsimple'
@@ -32,45 +33,68 @@ module Aspera
         ATOM_EXT_PARAMS = [MAX_ITEMS, MAX_PAGES].concat(ATOM_PARAMS).freeze
         # sub path in url for public link delivery
         PUB_LINK_EXTERNAL_MATCH = 'external_deliveries/'
+        STANDARD_PATH = '/aspera/faspex'
         private_constant(*%i[KEY_NODE KEY_PATH PACKAGE_MATCH_FIELD ATOM_MAILBOXES ATOM_PARAMS ATOM_EXT_PARAMS PUB_LINK_EXTERNAL_MATCH])
 
         class << self
-          def detect(base_url)
-            api = Rest.new(base_url: base_url)
-            result = api.call(
-              operation:        'POST',
-              subpath:          'aspera/faspex',
-              headers:          {'Accept' => 'application/xrds+xml', 'Content-type' => 'text/plain'},
-              text_body_params: '')
-            # 4.x
-            if result[:http].body.start_with?('<?xml')
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+
+            urls.each do |base_url|
+              next unless base_url.start_with?('https://')
+              api = Rest.new(base_url: base_url, redirect_max: 1)
+              result = api.call(
+                operation:        'POST',
+                subpath:          '',
+                headers:          {'Accept' => 'application/xrds+xml', 'Content-type' => 'text/plain'},
+                text_body_params: '')
+              # 4.x
+              next unless result[:http].body.start_with?('<?xml')
               res_s = XmlSimple.xml_in(result[:http].body, {'ForceArray' => false})
+              Log.log.debug{"version: #{result[:http]['X-IBM-Aspera']}"}
               version = res_s['XRD']['application']['version']
-              return {version: version, url: result[:http].uri}
+              # take redirect if any
+              return {version: version, url: result[:http].uri.to_s}
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
             end
             return nil
           end
 
+          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+            options = object.options
+            return {
+              preset_value: {
+                url:      options.get_option(:url, mandatory: true),
+                username: options.get_option(:username, mandatory: true),
+                password: options.get_option(:password, mandatory: true)
+              },
+              test_args:    'me'
+            }
+          end
+
           # extract elements from anonymous faspex link
           def get_link_data(public_url)
             public_uri = URI.parse(public_url)
-            raise CliBadArgument, 'Public link does not match Faspex format' unless (m = public_uri.path.match(%r{^(.*)/(external.*)$}))
+            raise Cli::BadArgument, 'Public link does not match Faspex format' unless (m = public_uri.path.match(%r{^(.*)/(external.*)$}))
             base = m[1]
             subpath = m[2]
             port_add = public_uri.port.eql?(public_uri.default_port) ? '' : ":#{public_uri.port}"
             result = {
               base_url: "#{public_uri.scheme}://#{public_uri.host}#{port_add}#{base}",
               subpath:  subpath,
-              query:    URI.decode_www_form(public_uri.query).each_with_object({}){|v, h|h[v.first] = v.last; }
+              query:    Rest.decode_query(public_uri.query)
             }
-            Log.dump('link data', result)
+            Log.log.debug{Log.dump('link data', result)}
             return result
           end
 
-          # get faspe: URI from entry in xml, and fix problems..
+          # get Fasp::Uri::SCHEME URI from entry in xml, and fix problems..
           def get_fasp_uri_from_entry(entry, raise_no_link: true)
             unless entry.key?('link')
-              raise CliBadArgument, 'package has no link (deleted?)' if raise_no_link
+              raise Cli::BadArgument, 'package has no link (deleted?)' if raise_no_link
               return nil
             end
             result = entry['link'].find{|e| e['rel'].eql?('package')}['href']
@@ -80,21 +104,11 @@ module Aspera
             return result
           end
 
-          def textify_package_list(table_data)
-            return table_data.map do |e|
-              e.each_key {|k| e[k] = e[k].first if e[k].is_a?(Array) && (e[k].length == 1)}
-              e['items'] = e.key?('link') ? e['link'].length : 0
-              e
-            end
-          end
-
-          # field_sym : :id or :name
-          def get_source_id(source_list, source_name)
-            source_ids = source_list.select { |i| i['name'].eql?(source_name) }
-            if source_ids.empty?
-              raise CliError, %Q(No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q("#{i['name']}")}.join(', ')}])
-            end
-            return source_ids.first['id']
+          # @return [Integer] identifier of source
+          def get_source_id_by_name(source_name, source_list)
+            match_source = source_list.find { |i| i['name'].eql?(source_name) }
+            return match_source['id'] unless match_source.nil?
+            raise Cli::Error, %Q(No such Faspex source: "#{source_name}" in [#{source_list.map{|i| %Q("#{i['name']}")}.join(', ')}])
           end
         end
 
@@ -104,8 +118,8 @@ module Aspera
           super(env)
           options.declare(:link, 'Public link for specific operation')
           options.declare(:delivery_info, 'Package delivery information', types: Hash)
-          options.declare(:source_name, 'Create package from remote source (by name)')
-          options.declare(:storage, 'Faspex local storage definition')
+          options.declare(:remote_source, 'Remote source for package send (id or %name:)')
+          options.declare(:storage, 'Faspex local storage definition (for browsing source)')
           options.declare(:recipient, 'Use if recipient is a dropbox (with *)')
           options.declare(:box, 'Package box', values: ATOM_MAILBOXES, default: :inbox)
           options.parse_options!
@@ -160,8 +174,8 @@ module Aspera
             # get a batch of package information
             # order: first batch is latest packages, and then in a batch ids are increasing
             atom_xml = api_v3.call({operation: 'GET', subpath: "#{mailbox}.atom", headers: {'Accept' => 'application/xml'}, url_params: mailbox_query})[:http].body
-            box_data = XmlSimple.xml_in(atom_xml, {'ForceArray' => true})
-            Log.dump(:box_data, box_data)
+            box_data = XmlSimple.xml_in(atom_xml, {'ForceArray' => %w[entry field link to]})
+            Log.log.debug{Log.dump(:box_data, box_data)}
             items = box_data.key?('entry') ? box_data['entry'] : []
             Log.log.debug{"new items: #{items.count}"}
             # it is the end if page is empty
@@ -173,11 +187,14 @@ module Aspera
               package[PACKAGE_MATCH_FIELD] =
                 case mailbox
                 when :inbox, :archive
-                  recipient = package['to'].find{|i|recipient_names.include?(i['name'].first)}
-                  recipient.nil? ? nil : recipient['recipient_delivery_id'].first
+                  recipient = package['to'].find{|i|recipient_names.include?(i['name'])}
+                  recipient.nil? ? nil : recipient['recipient_delivery_id']
                 else # :sent
-                  package['delivery_id'].first
+                  package['delivery_id']
                 end
+              # add special key
+              package['items'] = package['link'].is_a?(Array) ? package['link'].length : 0
+              package['metadata'] = package['metadata']['field'].each_with_object({}){|i, m| m[i['name']] = i['content'] }
               # if we look for a specific package
               stop_condition = true if !stop_at_id.nil? && stop_at_id.eql?(package[PACKAGE_MATCH_FIELD])
               # keep only those for the specified recipient
@@ -197,7 +214,7 @@ module Aspera
             break if link.nil?
             # replace parameters with the ones from next link
             params = CGI.parse(URI.parse(link['href']).query)
-            mailbox_query = params.keys.each_with_object({}){|i, m|; m[i] = params[i].first; }
+            mailbox_query = params.keys.each_with_object({}){|i, m| m[i] = params[i].first }
             Log.log.debug{"query: #{mailbox_query}"}
             break if !max_pages.nil? && (mailbox_query['page'].to_i > max_pages)
           end
@@ -210,7 +227,7 @@ module Aspera
           # pub link user
           link_data = self.class.get_link_data(public_link_url)
           if !['external/submissions/new', 'external/dropbox_submissions/new'].include?(link_data[:subpath])
-            raise CliBadArgument, "pub link is #{link_data[:subpath]}, expecting external/submissions/new"
+            raise Cli::BadArgument, "pub link is #{link_data[:subpath]}, expecting external/submissions/new"
           end
           create_path = link_data[:subpath].split('/')[0..-2].join('/')
           package_create_params[:passcode] = link_data[:query]['passcode']
@@ -225,11 +242,11 @@ module Aspera
             subpath:     create_path,
             json_params: package_create_params,
             headers:     {'Accept' => 'text/javascript'}})[:http].body
-          # get args of function call
+          # get arguments of function call
           package_creation_data.delete!("\n") # one line
           package_creation_data.gsub!(/^[^"]+\("\{/, '{') # delete header
           package_creation_data.gsub!(/"\);[^"]+$/, '"') # delete trailer
-          package_creation_data.gsub!(/\}", *"/, '},"') # between two args
+          package_creation_data.gsub!(/\}", *"/, '},"') # between two arguments
           package_creation_data.gsub!('\\"', '"') # remove protecting quote
           begin
             package_creation_data = JSON.parse("[#{package_creation_data}]")
@@ -254,18 +271,20 @@ module Aspera
             end
             return nagios.result
           when :package
-            command_pkg = options.get_next_command(%i[send recv list])
+            command_pkg = options.get_next_command(%i[send recv list show])
             case command_pkg
+            when :show
+              delivery_id = instance_identifier
+              return {type: :single_object, data: mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)} }
             when :list
               return {
-                type:    :object_list,
-                data:    mailbox_filtered_entries,
-                fields:  [PACKAGE_MATCH_FIELD, 'title', 'items'],
-                textify: lambda {|table_data|Faspex.textify_package_list(table_data)}
+                type:   :object_list,
+                data:   mailbox_filtered_entries,
+                fields: [PACKAGE_MATCH_FIELD, 'title', 'items']
               }
             when :send
               delivery_info = options.get_option(:delivery_info, mandatory: true)
-              raise CliBadArgument, 'delivery_info must be hash, refer to doc' unless delivery_info.is_a?(Hash)
+              raise Cli::BadArgument, 'delivery_info must be hash, refer to doc' unless delivery_info.is_a?(Hash)
               # actual parameter to faspex API
               package_create_params = {'delivery' => delivery_info}
               public_link_url = options.get_option(:link)
@@ -274,30 +293,30 @@ module Aspera
                 delivery_info['sources'] ||= [{'paths' => []}]
                 first_source = delivery_info['sources'].first
                 first_source['paths'].push(*transfer.source_list)
-                source_name = options.get_option(:source_name)
-                if !source_name.nil?
+                source_id = instance_identifier(as_option: :remote_source) do |field, value|
+                  raise Cli::BadArgument, 'only name as selector, or give id' unless field.eql?('name')
                   source_list = api_v3.call({operation: 'GET', subpath: 'source_shares', headers: {'Accept' => 'application/json'}})[:data]['items']
-                  source_id = self.class.get_source_id(source_list, source_name)
-                  first_source['id'] = source_id
+                  self.class.get_source_id_by_name(value, source_list)
                 end
+                first_source['id'] = source_id.to_i unless source_id.nil?
                 pkg_created = api_v3.call({
                   operation:   'POST',
                   subpath:     'send',
                   json_params: package_create_params,
                   headers:     {'Accept' => 'application/json'}
                 })[:data]
-                if !source_name.nil?
-                  # no transfer spec if remote source
+                if first_source.key?('id')
+                  # no transfer spec if remote source: handled by faspex
                   return {data: [pkg_created['links']['status']], type: :value_list, name: 'link'}
                 end
-                raise CliBadArgument, 'expecting one session exactly' if pkg_created['xfer_sessions'].length != 1
+                raise Cli::BadArgument, 'expecting one session exactly' if pkg_created['xfer_sessions'].length != 1
                 transfer_spec = pkg_created['xfer_sessions'].first
                 # use source from cmd line, this one only contains destination (already in dest root)
                 transfer_spec.delete('paths')
               else # public link
                 transfer_spec = send_public_link_to_ts(public_link_url, package_create_params)
               end
-              # Log.dump('transfer_spec',transfer_spec)
+              # Log.log.debug{Log.dump('transfer_spec',transfer_spec)}
               return Main.result_transfer(transfer.start(transfer_spec))
             when :recv
               link_url = options.get_option(:link)
@@ -322,11 +341,11 @@ module Aspera
                 delivery_id = instance_identifier
                 raise 'empty id' if delivery_id.empty?
                 recipient = options.get_option(:recipient)
-                if VAL_ALL.eql?(delivery_id)
+                if ExtendedValue::ALL.eql?(delivery_id)
                   pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD], uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
                 elsif !recipient.nil? && recipient.start_with?('*')
                   found_package_link = mailbox_filtered_entries(stop_at_id: delivery_id).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivery_id)}['link'].first['href']
-                  raise 'Not Found. Dropbox and Workgroup packages can use the link option with faspe:' if found_package_link.nil?
+                  raise "Not Found. Dropbox and Workgroup packages can use the link option with #{Fasp::Uri::SCHEME}" if found_package_link.nil?
                   pkg_id_uri = [{id: delivery_id, uri: found_package_link}]
                 else
                   # TODO: delivery id is the right one if package was receive by workgroup
@@ -339,12 +358,12 @@ module Aspera
                   package_entry = XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
                   pkg_id_uri = [{id: delivery_id, uri: self.class.get_fasp_uri_from_entry(package_entry)}]
                 end
-              when /^faspe:/
+              when /^#{Fasp::Uri::SCHEME}:/o
                 pkg_id_uri = [{id: 'package', uri: link_url}]
               else
                 link_data = self.class.get_link_data(link_url)
                 if !link_data[:subpath].start_with?(PUB_LINK_EXTERNAL_MATCH)
-                  raise CliBadArgument, "Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
+                  raise Cli::BadArgument, "Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
                 end
                 # NOTE: unauthenticated API (authorization is in url params)
                 api_public_link = Rest.new({base_url: link_data[:base_url]})
@@ -355,10 +374,10 @@ module Aspera
                   headers: {'Accept' => 'application/xml'})
                 if !package_creation_data[:http].body.start_with?('<?xml ')
                   OpenApplication.instance.uri(link_url)
-                  raise CliError, 'Unexpected response: package not found ?'
+                  raise Cli::Error, 'Unexpected response: package not found ?'
                 end
                 package_entry = XmlSimple.xml_in(package_creation_data[:http].body, {'ForceArray' => false})
-                Log.dump(:package_entry, package_entry)
+                Log.log.debug{Log.dump(:package_entry, package_entry)}
                 transfer_uri = self.class.get_fasp_uri_from_entry(package_entry)
                 pkg_id_uri = [{id: package_entry['id'], uri: transfer_uri}]
               end # public link
@@ -366,7 +385,7 @@ module Aspera
               # TODO : remove ids from skip not present in inbox to avoid growing too big
               # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
               pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
-              Log.dump(:pkg_id_uri, pkg_id_uri)
+              Log.log.debug{Log.dump(:pkg_id_uri, pkg_id_uri)}
               return Main.result_status('no new package') if pkg_id_uri.empty?
               result_transfer = []
               pkg_id_uri.each do |id_uri|
@@ -375,7 +394,7 @@ module Aspera
                   statuses = [:success]
                 else
                   transfer_spec = Fasp::Uri.new(id_uri[:uri]).transfer_spec
-                  # NOTE: only external users have token in faspe: link !
+                  # NOTE: only external users have token in Fasp::Uri::SCHEME link !
                   if !transfer_spec.key?('token')
                     sanitized = id_uri[:uri].gsub('&', '&amp;')
                     xml_payload =
@@ -397,41 +416,37 @@ module Aspera
               return Main.result_transfer_multiple(result_transfer)
             end
           when :source
-            command_source = options.get_next_command(%i[list id name])
+            command_source = options.get_next_command(%i[list info node])
             source_list = api_v3.call({operation: 'GET', subpath: 'source_shares', headers: {'Accept' => 'application/json'}})[:data]['items']
             case command_source
             when :list
               return {type: :object_list, data: source_list}
-            else # :id or :name
-              source_match_val = options.get_next_argument('source id or name')
-              source_ids = source_list.select { |i| i[command_source.to_s].to_s.eql?(source_match_val) }
-              if source_ids.empty?
-                raise CliError, "No such Faspex source #{command_source}: #{source_match_val} in [#{source_list.map{|i| i[command_source.to_s]}.join(', ')}]"
-              end
-              # get id and name
-              source_name = source_ids.first['name']
-              # source_id=source_ids.first['id']
+            else # :info :node
+              source_id = instance_identifier do |field, value|
+                raise Cli::BadArgument, 'only name as selector, or give id' unless field.eql?('name')
+                self.class.get_source_id_by_name(value, source_list)
+              end.to_i
+              source_name = source_list.find{|i|i['id'].eql?(source_id)}['name']
               source_hash = options.get_option(:storage, mandatory: true)
               # check value of option
-              raise CliError, 'storage option must be a Hash' unless source_hash.is_a?(Hash)
+              raise Cli::Error, 'storage option must be a Hash' unless source_hash.is_a?(Hash)
               source_hash.each do |name, storage|
-                raise CliError, "storage '#{name}' must be a Hash" unless storage.is_a?(Hash)
+                raise Cli::Error, "storage '#{name}' must be a Hash" unless storage.is_a?(Hash)
                 [KEY_NODE, KEY_PATH].each do |key|
-                  raise CliError, "storage '#{name}' must have a '#{key}'" unless storage.key?(key)
+                  raise Cli::Error, "storage '#{name}' must have a '#{key}'" unless storage.key?(key)
                 end
               end
               if !source_hash.key?(source_name)
-                raise CliError, "No such storage in config file: \"#{source_name}\" in [#{source_hash.keys.join(', ')}]"
+                raise Cli::Error, "No such storage in config file: \"#{source_name}\" in [#{source_hash.keys.join(', ')}]"
               end
               source_info = source_hash[source_name]
-              Log.log.debug{"source_info: #{source_info}"}
-              command_node = options.get_next_command(%i[info node])
-              case command_node
+              Log.log.debug{Log.dump(:source_info, source_info)}
+              case command_source
               when :info
                 return {data: source_info, type: :single_object}
               when :node
                 node_config = ExtendedValue.instance.evaluate(source_info[KEY_NODE])
-                raise CliError, "bad type for: \"#{source_info[KEY_NODE]}\"" unless node_config.is_a?(Hash)
+                raise Cli::Error, "bad type for: \"#{source_info[KEY_NODE]}\"" unless node_config.is_a?(Hash)
                 Log.log.debug{"node=#{node_config}"}
                 api_node = Rest.new({
                   base_url: node_config['url'],
@@ -440,7 +455,7 @@ module Aspera
                     username: node_config['username'],
                     password: node_config['password']}})
                 command = options.get_next_command(Node::COMMANDS_FASPEX)
-                return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_action(command, source_info[KEY_PATH])
+                return Node.new(@agents, api: api_node).execute_action(command, source_info[KEY_PATH])
               end
             end
           when :me