aspera-cli 4.14.0 → 4.16.0

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,14 +1,15 @@
 # frozen_string_literal: true
 
-# spellchecker: ignore workgroups,mypackages
+# spellchecker: ignore workgroups mypackages passcode
 
 require 'aspera/cli/basic_auth_plugin'
+require 'aspera/cli/extended_value'
 require 'aspera/persistency_action_once'
 require 'aspera/id_generator'
 require 'aspera/nagios'
 require 'aspera/environment'
+require 'aspera/assert'
 require 'securerandom'
-require 'ruby-progressbar'
 require 'tty-spinner'
 
 module Aspera
@@ -20,80 +21,132 @@ module Aspera
         API_DETECT = 'api/v5/configuration/ping'
         # list of supported mailbox types (to list packages)
         API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
-        PACKAGE_ALL_INIT = 'INIT'
         PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
-        ADMIN_RESOURCES = %i[accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs metadata_profiles
-                             email_notifications].freeze
-        private_constant(*%i[RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE])
+        # Faspex API v5: get transfer spec for connect
+        TRANSFER_CONNECT = 'connect'
+        ADMIN_RESOURCES = %i[
+          accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
+          metadata_profiles email_notifications alternate_addresses
+        ].freeze
+        JOB_RUNNING = %w[queued working].freeze
+        STANDARD_PATH = '/aspera/faspex'
+        PER_PAGE_DEFAULT = 100
+        private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE PER_PAGE_DEFAULT])
         class << self
-          def detect(base_url)
-            api = Rest.new(base_url: base_url, redirect_max: 1)
-            result = api.read(API_DETECT)
-            if result[:http].code.start_with?('2') && result[:http].body.strip.empty?
-              suffix_length = -2 - API_DETECT.length
+          def application_name
+            'Faspex'
+          end
+
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+
+            urls.each do |base_url|
+              next unless base_url.start_with?('https://')
+              api = Rest.new(base_url: base_url, redirect_max: 1)
+              result = api.read(API_DETECT)
+              next unless result[:http].code.start_with?('2') && result[:http].body.strip.empty?
+              url_length = -2 - API_DETECT.length
+              # take redirect if any
               return {
                 version: result[:http]['x-ibm-aspera'] || '5',
-                url:     result[:http].uri.to_s[0..suffix_length],
-                name:    'Faspex 5'
+                url:     result[:http].uri.to_s[0..url_length]
               }
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
             end
             return nil
           end
-        end
 
-        # Faspex API v5: get transfer spec for connect
-        TRANSFER_CONNECT = 'connect'
+          def wizard(object:, private_key_path:, pub_key_pem:)
+            options = object.options
+            formatter = object.formatter
+            instance_url = options.get_option(:url, mandatory: true)
+            wiz_username = options.get_option(:username, mandatory: true)
+            raise "Username shall be an email in Faspex: #{wiz_username}" if !(wiz_username =~ /\A[\w+\-.]+@[a-z\d\-.]+\.[a-z]+\z/i)
+            if options.get_option(:client_id).nil? || options.get_option(:client_secret).nil?
+              formatter.display_status('Ask the ascli client id and secret to your Administrator.'.red)
+              formatter.display_status("Admin should login to: #{instance_url}")
+              OpenApplication.instance.uri(instance_url)
+              formatter.display_status('Navigate to: 𓃑  → Admin → Configurations → API clients')
+              formatter.display_status('Create an API client with:')
+              formatter.display_status('- name: ascli')
+              formatter.display_status('- JWT: enabled')
+              formatter.display_status("Then, logged in as #{wiz_username.red} go to your profile:")
+              formatter.display_status('👤 → Account Settings → Preferences -> Public Key in PEM:')
+              formatter.display_status(pub_key_pem)
+              formatter.display_status('Once set, fill in the parameters:')
+            end
+            return {
+              preset_value: {
+                url:           instance_url,
+                username:      wiz_username,
+                auth:          :jwt.to_s,
+                private_key:   "@file:#{private_key_path}",
+                client_id:     options.get_option(:client_id, mandatory: true),
+                client_secret: options.get_option(:client_secret, mandatory: true)
+              },
+              test_args:    'user profile show'
+            }
+          end
+
+          def public_link?(url)
+            url.include?('/public/')
+          end
+        end
 
         def initialize(env)
           super(env)
           options.declare(:client_id, 'OAuth client identifier')
           options.declare(:client_secret, 'OAuth client secret')
           options.declare(:redirect_uri, 'OAuth redirect URI for web authentication')
-          options.declare(:auth, 'OAuth type of authentication', values: %i[boot link].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
+          options.declare(:auth, 'OAuth type of authentication', values: %i[boot].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
           options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
           options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
-          options.declare(:link, 'Public link authorization (specific operations)')
-          options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{VAL_ALL}", default: 'inbox')
+          options.declare(:box, "Package inbox, either shared inbox name or one of: #{API_LIST_MAILBOX_TYPES.join(', ')} or #{ExtendedValue::ALL}", default: 'inbox')
           options.declare(:shared_folder, 'Send package with files from shared folder')
-          options.declare(:group_type, 'Shared inbox or workgroup', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
+          options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
           options.parse_options!
         end
 
+        def api_url
+          return "#{@faspex5_api_base_url}/api/v5"
+        end
+
+        def auth_api_url
+          return "#{@faspex5_api_base_url}/auth"
+        end
+
         def set_api
-          public_link = options.get_option(:link)
-          unless public_link.nil?
-            @faspex5_api_base_url = public_link.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
-            options.set_option(:auth, :link)
-          end
-          @faspex5_api_base_url ||= options.get_option(:url, mandatory: true).gsub(%r{/+$}, '')
-          @faspex5_api_auth_url = "#{@faspex5_api_base_url}/auth"
-          faspex5_api_v5_url = "#{@faspex5_api_base_url}/api/v5"
-          case options.get_option(:auth, mandatory: true)
-          when :link
-            uri = URI.parse(public_link)
-            args = URI.decode_www_form(uri.query).each_with_object({}){|v, h|h[v.first] = v.last; }
-            Log.dump(:args, args)
-            context = args['context']
-            raise 'missing context' if context.nil?
-            @pub_link_context = JSON.parse(Base64.decode64(context))
-            Log.dump(:@pub_link_context, @pub_link_context)
+          # get endpoint, remove unnecessary trailing slashes
+          @faspex5_api_base_url = options.get_option(:url, mandatory: true).gsub(%r{/+$}, '')
+          auth_type = self.class.public_link?(@faspex5_api_base_url) ? :public_link : options.get_option(:auth, mandatory: true)
+          case auth_type
+          when :public_link
+            encoded_context = Rest.decode_query(URI.parse(@faspex5_api_base_url).query)['context']
+            raise 'Bad faspex5 public link, missing context in query' if encoded_context.nil?
+            @pub_link_context = JSON.parse(Base64.decode64(encoded_context))
+            Log.log.trace1{Log.dump(:@pub_link_context, @pub_link_context)}
+            # ok, we have the additional parameters, get the base url
+            @faspex5_api_base_url = @faspex5_api_base_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               headers:  {'Passcode' => @pub_link_context['passcode']}
             })
           when :boot
             # the password here is the token copied directly from browser in developer mode
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               headers:  {'Authorization' => options.get_option(:password, mandatory: true)}
             })
           when :web
             # opens a browser and ask user to auth using web
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               auth:     {
                 type:         :oauth2,
-                base_url:     @faspex5_api_auth_url,
+                base_url:     auth_api_url,
                 grant_method: :web,
                 client_id:    options.get_option(:client_id, mandatory: true),
                 web:          {redirect_uri: options.get_option(:redirect_uri, mandatory: true)}
@@ -101,10 +154,10 @@ module Aspera
           when :jwt
             app_client_id = options.get_option(:client_id, mandatory: true)
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               auth:     {
                 type:         :oauth2,
-                base_url:     @faspex5_api_auth_url,
+                base_url:     auth_api_url,
                 grant_method: :jwt,
                 client_id:    app_client_id,
                 jwt:          {
@@ -117,14 +170,14 @@ module Aspera
                   headers:         {typ: 'JWT'}
                 }
               }})
-          else raise 'Unexpected case for option: auth'
+          else error_unexpected_value(auth_type)
           end
         end
 
         # if recipient is just an email, then convert to expected API hash : name and type
         def normalize_recipients(parameters)
           return unless parameters.key?('recipients')
-          raise 'Field recipients must be an Array' unless parameters['recipients'].is_a?(Array)
+          assert_type(parameters['recipients'], Array){'recipients'}
           recipient_types = RECIPIENT_TYPES
           if parameters.key?('recipient_types')
             recipient_types = parameters['recipient_types']
@@ -147,52 +200,63 @@ module Aspera
 
         # wait for package status to be in provided list
         def wait_package_status(id, status_list: PACKAGE_TERMINATED)
-          spinner = nil
-          progress = nil
-          while true
+          total_sent = false
+          loop do
             status = @api_v5.read("packages/#{id}/upload_details")[:data]
+            status['id'] = id
             # user asked to not follow
-            break if status_list.nil? || status_list.include?(status['upload_status'])
+            return status if status_list.nil?
             if status['upload_status'].eql?('submitted')
-              if spinner.nil?
-                spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
-                spinner.start
-              end
-              spinner.update(title: status['upload_status'])
-              spinner.spin
-            elsif progress.nil?
-              progress = ProgressBar.create(
-                format:     '%a %B %p%% %r Mbps %e',
-                rate_scale: lambda{|rate|rate / Environment::BYTES_PER_MEBIBIT},
-                title:      'progress',
-                total:      status['bytes_total'].to_i)
+              config.progress_bar&.event(session_id: nil, type: :pre_start, info: status['upload_status'])
+            elsif !total_sent
+              config.progress_bar&.event(session_id: id, type: :session_start)
+              config.progress_bar&.event(session_id: id, type: :session_size, info: status['bytes_total'].to_i)
+              total_sent = true
             else
-              progress.progress = status['bytes_written'].to_i
+              config.progress_bar&.event(session_id: id, type: :transfer, info: status['bytes_written'].to_i)
+            end
+            if status_list.include?(status['upload_status'])
+              # if status['upload_status'].eql?('completed')
+              config.progress_bar&.event(session_id: id, type: :end)
+              return status
+              # end
             end
+            sleep(1.0)
+          end
+        end
+
+        def wait_for_job(job_id)
+          spinner = nil
+          loop do
+            status = @api_v5.read("jobs/#{job_id}", {type: :formatted})[:data]
+            return status unless JOB_RUNNING.include?(status['status'])
+            if spinner.nil?
+              spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
+              spinner.start
+            end
+            spinner.update(title: status['status'])
+            spinner.spin
             sleep(0.5)
           end
-          status['id'] = id
-          return status
+          error_unreachable_line
         end
 
-        # get a (full or partial) list of all entities of a given type
+        # Get a (full or partial) list of all entities of a given type
         # @param type [String] the type of entity to list (just a name)
         # @param query [Hash,nil] additional query parameters
-        # @param path [String] optional prefix to add to the path (nil or empty string: no prefix)
+        # @param real_path [String] real path if it's n ot just the type
         # @param item_list_key [String] key in the result to get the list of items
-        def list_entities(type:, path: nil, query: nil, item_list_key: nil)
-          query = {} if query.nil?
+        def list_entities(type:, real_path: nil, query: {}, item_list_key: nil)
           type = type.to_s if type.is_a?(Symbol)
+          assert_type(type, String)
           item_list_key = type if item_list_key.nil?
-          raise "internal error: Invalid type #{type.class}" unless type.is_a?(String)
-          full_path = type
-          full_path = "#{path}/#{full_path}" unless path.nil? || path.empty?
+          full_path = real_path.nil? ? type : real_path
           result = []
           offset = 0
           max_items = query.delete(MAX_ITEMS)
           remain_pages = query.delete(MAX_PAGES)
           # merge default parameters, by default 100 per page
-          query = {'limit'=> 100}.merge(query)
+          query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
           loop do
             query['offset'] = offset
             page_result = @api_v5.read(full_path, query)[:data]
@@ -211,36 +275,39 @@ module Aspera
         end
 
         # lookup an entity id from its name
-        def lookup_entity_by_field(type:, value:, field: 'name', query: :default, path: nil, item_list_key: nil)
-          query = {'q'=> value} if query.eql?(:default)
-          found = list_entities(type: type, path: path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
+        def lookup_entity_by_field(type:, value:, field: 'name', query: :default, real_path: nil, item_list_key: nil)
+          if query.eql?(:default)
+            assert(field.eql?('name')){'Default query is on name only'}
+            query = {'q'=> value}
+          end
+          found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
           case found.length
           when 0 then raise "No #{type} with #{field} = #{value}"
           when 1 then return found.first
-          else raise "Found #{found.length} #{path} with #{field} = #{value}"
+          else raise "Found #{found.length} #{real_path} with #{field} = #{value}"
           end
         end
 
         # list all packages with optional filter
-        def list_packages_with_filter
+        def list_packages_with_filter(query: {})
           filter = options.get_next_argument('filter', mandatory: false, type: Proc, default: ->(_x){true})
           # translate box name to API prefix (with ending slash)
           box = options.get_option(:box)
-          api_path =
+          real_path =
             case box
-            when VAL_ALL then '' # only admin can list all packages globally
-            when *API_LIST_MAILBOX_TYPES then box
+            when ExtendedValue::ALL then 'packages' # only admin can list all packages globally
+            when *API_LIST_MAILBOX_TYPES then "#{box}/packages"
             else
               group_type = options.get_option(:group_type)
-              "#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}"
+              "#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}/packages"
             end
           return list_entities(
             type: 'packages',
-            query:  query_read_delete(default: {}),
-            path: api_path).select(&filter)
+            query:  query_read_delete(default: query),
+            real_path: real_path).select(&filter)
         end
 
-        def package_receive
+        def package_receive(package_ids)
           # prepare persistency if needed
           skip_ids_persistency = nil
           if options.get_option(:once_only, mandatory: true)
@@ -251,52 +318,53 @@ module Aspera
               id:      IdGenerator.from_list([
                 'faspex_recv',
                 options.get_option(:url, mandatory: true),
-                options.get_option(:username, mandatory: true)]))
+                options.get_option(:username, mandatory: true),
+                options.get_option(:box, mandatory: true)
+              ]))
           end
-          package_ids =
-            if @pub_link_context&.key?('package_id')
-              @pub_link_context['package_id']
-            else
-              # one or several packages
-              instance_identifier
-            end
+          packages = []
           case package_ids
-          when PACKAGE_ALL_INIT
-            raise 'Only with option once_only' unless skip_ids_persistency
+          when ExtendedValue::INIT
+            assert(skip_ids_persistency){'Only with option once_only'}
             skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
             skip_ids_persistency.save
             return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
-          when VAL_ALL
+          when ExtendedValue::ALL
             # TODO: if packages have same name, they will overwrite ?
-            package_ids = list_packages_with_filter.map{|p|p['id']}
-            Log.dump(:package_ids, package_ids)
-            Log.dump(:package_ids, skip_ids_persistency.data)
-            package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
-            Log.dump(:package_ids, package_ids)
+            packages = list_packages_with_filter(query: {'status' => 'completed'})
+            Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
+            Log.log.trace1{Log.dump(:skip_ids, skip_ids_persistency.data)}
+            packages.reject!{|p|skip_ids_persistency.data.include?(p['id'])} if skip_ids_persistency
+            Log.log.trace1{Log.dump(:package_ids, packages.map{|p|p['id']})}
+          else
+            # a single id was provided, or a list of ids
+            package_ids = [package_ids] unless package_ids.is_a?(Array)
+            assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
+            assert(package_ids.all?(String)){'Package id shall be String'}
+            # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")[:data]}
+            packages = package_ids.map{|pkg_id|{'id'=>pkg_id}}
           end
-          # a single id was provided
-          # TODO: check package_ids is a list of strings
-          package_ids = [package_ids] if package_ids.is_a?(String)
           result_transfer = []
-          package_ids.each do |pkg_id|
+          param_file_list = {}
+          begin
+            param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
+          rescue Cli::BadArgument
+            # paths is optional
+          end
+          download_params = {
+            type:          'received',
+            transfer_type: TRANSFER_CONNECT
+          }
+          box = options.get_option(:box)
+          case box
+          when /outbox/ then download_params[:type] = 'sent'
+          when *API_LIST_MAILBOX_TYPES then nil # nothing to do
+          else # shared inbox / workgroup
+            download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
+          end
+          packages.each do |package|
+            pkg_id = package['id']
             formatter.display_status("Receiving package #{pkg_id}")
-            param_file_list = {}
-            begin
-              param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
-            rescue Aspera::Cli::CliBadArgument
-              # paths is optional
-            end
-            download_params = {
-              type:          'received',
-              transfer_type: TRANSFER_CONNECT
-            }
-            box = options.get_option(:box)
-            case box
-            when /outbox/ then download_params[:type] = 'sent'
-            when *API_LIST_MAILBOX_TYPES then nil # nothing to do
-            else # shared inbox / workgroup
-              download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
-            end
             # TODO: allow from sent as well ?
             transfer_spec = @api_v5.call(
               operation:   'POST',
@@ -319,21 +387,15 @@ module Aspera
         end
 
         def package_action
-          command = options.get_next_command(%i[list show browse status delete send receive])
+          command = options.get_next_command(%i[show browse status delete receive send list])
+          package_id =
+            if %i[receive show browse status delete].include?(command)
+              @pub_link_context&.key?('package_id') ? @pub_link_context['package_id'] : instance_identifier
+            end
           case command
-          when :list
-            return {
-              type:   :object_list,
-              data:   list_packages_with_filter,
-              fields: %w[id title release_date total_bytes total_files created_time state]
-            }
           when :show
-            id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-            id ||= instance_identifier
-            return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
+            return {type: :single_object, data: @api_v5.read("packages/#{package_id}")[:data]}
           when :browse
-            id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-            id ||= instance_identifier
             path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
             # TODO: support multi-page listing ?
             params = {
@@ -343,29 +405,32 @@ module Aspera
             }
             result = @api_v5.call({
               operation:   'POST',
-              subpath:     "packages/#{id}/files/received",
+              subpath:     "packages/#{package_id}/files/received",
               headers:     {'Accept' => 'application/json'},
               url_params:  params,
               json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
             formatter.display_item_count(result['item_count'], result['total_count'])
             return {type: :object_list, data: result['items']}
           when :status
-            status = wait_package_status(instance_identifier, status_list: nil)
+            status = wait_package_status(package_id, status_list: nil)
             return {type: :single_object, data: status}
           when :delete
-            ids = instance_identifier
+            ids = package_id
             ids = [ids] unless ids.is_a?(Array)
-            raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
+            assert_type(ids, Array){'Package identifier'}
+            assert(ids.all?(String)){'Package id shall be String'}
             # API returns 204, empty on success
             @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
             return Main.result_status('Package(s) deleted')
+          when :receive
+            return package_receive(package_id)
           when :send
-            parameters = value_create_modify(command: command, type: Hash)
+            parameters = value_create_modify(command: command)
             normalize_recipients(parameters)
             package = @api_v5.create('packages', parameters)[:data]
             shared_folder = options.get_option(:shared_folder)
             if shared_folder.nil?
-              # TODO: option to send from remote source or httpgw
+              # send from local files
               transfer_spec = @api_v5.call(
                 operation:   'POST',
                 subpath:     "packages/#{package['id']}/transfer_spec/upload",
@@ -377,6 +442,7 @@ module Aspera
               transfer_spec.delete('authentication')
               return Main.result_transfer(transfer.start(transfer_spec))
             else
+              # send from remote shared folder
               if (m = shared_folder.match(REGEX_LOOKUP_ID_BY_FIELD))
                 shared_folder = lookup_entity_by_field(type: 'shared_folders', value: m[2])['id']
               end
@@ -390,8 +456,12 @@ module Aspera
               end
               return {type: :single_object, data: result}
             end
-          when :receive
-            return package_receive
+          when :list
+            return {
+              type:   :object_list,
+              data:   list_packages_with_filter,
+              fields: %w[id title release_date total_bytes total_files created_time state]
+            }
           end # case package
         end
 
@@ -465,26 +535,44 @@ module Aspera
               id_as_arg = false
               display_fields = nil
               adm_api = @api_v5
+              special_query = :default
               available_commands = [].concat(Plugin::ALL_OPS)
               case res_type
               when :metadata_profiles
                 res_path = 'configuration/metadata_profiles'
                 list_key = 'profiles'
+              when :alternate_addresses
+                res_path = 'configuration/alternate_addresses'
               when :email_notifications
                 list_key = false
                 id_as_arg = 'type'
               when :accounts
-                display_fields = [:all_but, 'user_profile_data_attributes']
+                display_fields = Formatter.all_but('user_profile_data_attributes')
               when :oauth_clients
-                display_fields = [:all_but, 'public_key']
-                adm_api = Rest.new(@api_v5.params.merge({base_url: @faspex5_api_auth_url}))
+                display_fields = Formatter.all_but('public_key')
+                adm_api = Rest.new(@api_v5.params.merge({base_url: auth_api_url}))
               when :shared_inboxes, :workgroups
                 available_commands.push(:members, :saml_groups, :invite_external_collaborator)
+                special_query = {'all': true}
+              when :nodes
+                available_commands.push(:shared_folders)
               end
               res_command = options.get_next_command(available_commands)
               case res_command
               when *Plugin::ALL_OPS
-                return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
+                return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg) do |field, value|
+                  lookup_entity_by_field(
+                    type: res_type, real_path: res_path, field: field, value: value, query: special_query)['id']
+                end
+              when :shared_folders
+                node_id = instance_identifier do |field, value|
+                  lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']
+                end
+                sh_path = "#{res_path}/#{node_id}/shared_folders"
+                return entity_action(adm_api, sh_path, item_list_key: 'shared_folders') do |field, value|
+                         lookup_entity_by_field(
+                           type: 'shared_folders', real_path: sh_path, field: field, value: value)['id']
+                       end
               when :invite_external_collaborator
                 shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
                 creation_payload = value_create_modify(command: res_command, type: [Hash, String])
@@ -492,7 +580,9 @@ module Aspera
                 res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
                 result = adm_api.create(res_path, creation_payload)[:data]
                 formatter.display_status(result['message'])
-                result = lookup_entity_by_field(type: 'members', path: "#{res_type}/#{shared_inbox_id}", value: creation_payload['email_address'], query: {})
+                result = lookup_entity_by_field(
+                  type: 'members', real_path: "#{res_type}/#{shared_inbox_id}/members", value: creation_payload['email_address'],
+                  query: {})
                 return {type: :single_object, data: result}
               when :members, :saml_groups
                 res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
@@ -533,20 +623,23 @@ module Aspera
               when :show
                 return { type: :single_object, data: @api_v5.read(smtp_path)[:data] }
               when :create
-                return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd, type: Hash))[:data] }
+                return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd))[:data] }
               when :modify
-                return { type: :single_object, data: @api_v5.modify(smtp_path, value_create_modify(command: smtp_cmd, type: Hash))[:data] }
+                return { type: :single_object, data: @api_v5.update(smtp_path, value_create_modify(command: smtp_cmd))[:data] }
               when :delete
                 return { type: :single_object, data: @api_v5.delete(smtp_path)[:data] }
               when :test
                 test_data = options.get_next_argument('Email or test data, see API')
                 test_data = {test_email_recipient: test_data} if test_data.is_a?(String)
-                return { type: :single_object, data: @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data] }
+                creation = @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data]
+                result = wait_for_job(creation['job_id'])
+                result['serialized_args'] = JSON.parse(result['serialized_args']) rescue result['serialized_args']
+                return { type: :single_object, data: result }
               end
             end
           when :gateway
             require 'aspera/faspex_gw'
-            url = value_create_modify(type: String)
+            url = value_create_modify(command: command, description: 'listening url (e.g. https://localhost:12345)', type: String)
             uri = URI.parse(url)
             server = WebServerSimple.new(uri)
             server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
@@ -559,9 +652,9 @@ module Aspera
             return Main.result_status('Gateway terminated')
           when :postprocessing
             require 'aspera/faspex_postproc' # cspell:disable-line
-            parameters = value_create_modify(type: Hash)
+            parameters = value_create_modify(command: command)
             parameters = parameters.symbolize_keys
-            raise 'Missing key: url' unless parameters.key?(:url)
+            assert(parameters.key?(:url)){'Missing key: url'}
             uri = URI.parse(parameters[:url])
             parameters[:processing] ||= {}
             parameters[:processing][:root] = uri.path
@@ -576,36 +669,6 @@ module Aspera
             return Main.result_status('Gateway terminated')
           end # case command
         end # action
-
-        def wizard(params)
-          if params[:prepare]
-            # if not defined by user, generate unique name
-            params[:preset_name] ||= [params[:plugin_sym]].concat(URI.parse(params[:instance_url]).host.gsub(/[^a-z0-9.]/, '').split('.')).join('_')
-            params[:need_private_key] = true
-            return
-          end
-          formatter.display_status('Ask the ascli client id and secret to your Administrator, or ask them to go to:'.red)
-          OpenApplication.instance.uri(params[:instance_url])
-          formatter.display_status('Then: 𓃑  → Admin → Configurations → API clients')
-          formatter.display_status('Create an API client with:')
-          formatter.display_status('- name: ascli')
-          formatter.display_status('- JWT: enabled')
-          formatter.display_status('Then, logged in as user go to your profile:')
-          formatter.display_status('👤 → Account Settings → Preferences -> Public Key in PEM:')
-          formatter.display_status(params[:pub_key_pem])
-          formatter.display_status('Once set, fill in the parameters:')
-          return {
-            preset_value: {
-              url:           params[:instance_url],
-              username:      options.get_option(:username, mandatory: true),
-              auth:          :jwt.to_s,
-              private_key:   '@file:' + params[:private_key_path],
-              client_id:     options.get_option(:client_id, mandatory: true),
-              client_secret: options.get_option(:client_secret, mandatory: true)
-            },
-            test_args:    "#{params[:plugin_sym]} user profile show"
-          }
-        end
       end # Faspex5
     end # Plugins
   end # Cli