aspera-cli 4.13.0 → 4.15.0

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# spellchecker: ignore workgroups,mypackages
+# spellchecker: ignore workgroups mypackages passcode
 
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
@@ -8,7 +8,6 @@ require 'aspera/id_generator'
 require 'aspera/nagios'
 require 'aspera/environment'
 require 'securerandom'
-require 'ruby-progressbar'
 require 'tty-spinner'
 
 module Aspera
@@ -18,98 +17,154 @@ module Aspera
         RECIPIENT_TYPES = %w[user workgroup external_user distribution_list shared_inbox].freeze
         PACKAGE_TERMINATED = %w[completed failed].freeze
         API_DETECT = 'api/v5/configuration/ping'
-        # list of supported mailbox types
-        API_MAILBOXES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
-        PACKAGE_TYPE_RECEIVED = 'received'
+        # list of supported mailbox types (to list packages)
+        API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
         PACKAGE_ALL_INIT = 'INIT'
-        private_constant(*%i[RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_MAILBOXES PACKAGE_TYPE_RECEIVED])
+        PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
+        # Faspex API v5: get transfer spec for connect
+        TRANSFER_CONNECT = 'connect'
+        ADMIN_RESOURCES = %i[
+          accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
+          metadata_profiles email_notifications
+        ].freeze
+        JOB_RUNNING = %w[queued working].freeze
+        STANDARD_PATH = '/aspera/faspex'
+        private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE])
         class << self
-          def detect(base_url)
-            api = Rest.new(base_url: base_url, redirect_max: 1)
-            result = api.read(API_DETECT)
-            if result[:http].code.start_with?('2') && result[:http].body.strip.empty?
-              suffix_length = -2 - API_DETECT.length
+          def application_name
+            'Faspex'
+          end
+
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+
+            urls.each do |base_url|
+              next unless base_url.start_with?('https://')
+              api = Rest.new(base_url: base_url, redirect_max: 1)
+              result = api.read(API_DETECT)
+              next unless result[:http].code.start_with?('2') && result[:http].body.strip.empty?
+              url_length = -2 - API_DETECT.length
+              # take redirect if any
               return {
                 version: result[:http]['x-ibm-aspera'] || '5',
-                url:     result[:http].uri.to_s[0..suffix_length]}
+                url:     result[:http].uri.to_s[0..url_length]
+              }
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
             end
             return nil
           end
-        end
 
-        TRANSFER_CONNECT = 'connect'
+          def wizard(object:, private_key_path:, pub_key_pem:)
+            options = object.options
+            formatter = object.formatter
+            instance_url = options.get_option(:url, mandatory: true)
+            wiz_username = options.get_option(:username, mandatory: true)
+            raise "Username shall be an email in Faspex: #{wiz_username}" if !(wiz_username =~ /\A[\w+\-.]+@[a-z\d\-.]+\.[a-z]+\z/i)
+            if options.get_option(:client_id).nil? || options.get_option(:client_secret).nil?
+              formatter.display_status('Ask the ascli client id and secret to your Administrator.'.red)
+              formatter.display_status("Admin should login to: #{instance_url}")
+              OpenApplication.instance.uri(instance_url)
+              formatter.display_status('Navigate to: 𓃑  → Admin → Configurations → API clients')
+              formatter.display_status('Create an API client with:')
+              formatter.display_status('- name: ascli')
+              formatter.display_status('- JWT: enabled')
+              formatter.display_status("Then, logged in as #{wiz_username.red} go to your profile:")
+              formatter.display_status('👤 → Account Settings → Preferences -> Public Key in PEM:')
+              formatter.display_status(pub_key_pem)
+              formatter.display_status('Once set, fill in the parameters:')
+            end
+            return {
+              preset_value: {
+                url:           instance_url,
+                username:      wiz_username,
+                auth:          :jwt.to_s,
+                private_key:   "@file:#{private_key_path}",
+                client_id:     options.get_option(:client_id, mandatory: true),
+                client_secret: options.get_option(:client_secret, mandatory: true)
+              },
+              test_args:    'user profile show'
+            }
+          end
+
+          def public_link?(url)
+            url.include?('/public/')
+          end
+        end
 
         def initialize(env)
           super(env)
-          options.add_opt_simple(:client_id, 'OAuth client identifier')
-          options.add_opt_simple(:client_secret, 'OAuth client secret')
-          options.add_opt_simple(:redirect_uri, 'OAuth redirect URI for web authentication')
-          options.add_opt_list(:auth, %i[boot link].concat(Oauth::STD_AUTH_TYPES), 'OAuth type of authentication')
-          options.add_opt_simple(:box, "Package inbox, either shared inbox name or one of #{API_MAILBOXES}")
-          options.add_opt_simple(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
-          options.add_opt_simple(:passphrase, 'RSA private key passphrase')
-          options.add_opt_simple(:shared_folder, 'Shared folder source for package files')
-          options.add_opt_simple(:link, 'public link for specific operation')
-          options.set_option(:auth, :jwt)
-          options.set_option(:box, 'inbox')
+          options.declare(:client_id, 'OAuth client identifier')
+          options.declare(:client_secret, 'OAuth client secret')
+          options.declare(:redirect_uri, 'OAuth redirect URI for web authentication')
+          options.declare(:auth, 'OAuth type of authentication', values: %i[boot].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
+          options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
+          options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
+          options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{ExtendedValue::ALL}", default: 'inbox')
+          options.declare(:shared_folder, 'Send package with files from shared folder')
+          options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
           options.parse_options!
         end
 
+        def api_url
+          return "#{@faspex5_api_base_url}/api/v5"
+        end
+
+        def auth_api_url
+          return "#{@faspex5_api_base_url}/auth"
+        end
+
         def set_api
-          public_link = options.get_option(:link)
-          unless public_link.nil?
-            @faspex5_api_base_url = public_link.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
-            options.set_option(:auth, :link)
-          end
-          @faspex5_api_base_url ||= options.get_option(:url, is_type: :mandatory).gsub(%r{/+$}, '')
-          @faspex5_api_auth_url = "#{@faspex5_api_base_url}/auth"
-          faspex5_api_v5_url = "#{@faspex5_api_base_url}/api/v5"
-          case options.get_option(:auth, is_type: :mandatory)
-          when :link
-            uri = URI.parse(public_link)
-            args = URI.decode_www_form(uri.query).each_with_object({}){|v, h|h[v.first] = v.last; }
-            Log.dump(:args, args)
-            context = args['context']
-            raise 'missing context' if context.nil?
-            @pub_link_context = JSON.parse(Base64.decode64(context))
-            Log.dump(:@pub_link_context, @pub_link_context)
+          # get endpoint, remove unnecessary trailing slashes
+          @faspex5_api_base_url = options.get_option(:url, mandatory: true).gsub(%r{/+$}, '')
+          auth_type = self.class.public_link?(@faspex5_api_base_url) ? :public_link : options.get_option(:auth, mandatory: true)
+          case auth_type
+          when :public_link
+            encoded_context = Rest.decode_query(URI.parse(@faspex5_api_base_url).query)['context']
+            raise 'Bad faspex5 public link, missing context in query' if encoded_context.nil?
+            @pub_link_context = JSON.parse(Base64.decode64(encoded_context))
+            Log.log.trace1{Log.dump(:@pub_link_context, @pub_link_context)}
+            # ok, we have the additional parameters, get the base url
+            @faspex5_api_base_url = @faspex5_api_base_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               headers:  {'Passcode' => @pub_link_context['passcode']}
             })
           when :boot
             # the password here is the token copied directly from browser in developer mode
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
-              headers:  {'Authorization' => options.get_option(:password, is_type: :mandatory)}
+              base_url: api_url,
+              headers:  {'Authorization' => options.get_option(:password, mandatory: true)}
             })
           when :web
             # opens a browser and ask user to auth using web
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               auth:     {
                 type:         :oauth2,
-                base_url:     @faspex5_api_auth_url,
+                base_url:     auth_api_url,
                 grant_method: :web,
-                client_id:    options.get_option(:client_id, is_type: :mandatory),
-                web:          {redirect_uri: options.get_option(:redirect_uri, is_type: :mandatory)}
+                client_id:    options.get_option(:client_id, mandatory: true),
+                web:          {redirect_uri: options.get_option(:redirect_uri, mandatory: true)}
               }})
           when :jwt
-            app_client_id = options.get_option(:client_id, is_type: :mandatory)
+            app_client_id = options.get_option(:client_id, mandatory: true)
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               auth:     {
                 type:         :oauth2,
-                base_url:     @faspex5_api_auth_url,
+                base_url:     auth_api_url,
                 grant_method: :jwt,
                 client_id:    app_client_id,
                 jwt:          {
                   payload:         {
                     iss: app_client_id,    # issuer
                     aud: app_client_id,    # audience (this field is not clear...)
-                    sub: "user:#{options.get_option(:username, is_type: :mandatory)}" # subject is a user
+                    sub: "user:#{options.get_option(:username, mandatory: true)}" # subject is a user
                   },
-                  private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, is_type: :mandatory), options.get_option(:passphrase)),
+                  private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, mandatory: true), options.get_option(:passphrase)),
                   headers:         {typ: 'JWT'}
                 }
               }})
@@ -121,13 +176,16 @@ module Aspera
         def normalize_recipients(parameters)
           return unless parameters.key?('recipients')
           raise 'Field recipients must be an Array' unless parameters['recipients'].is_a?(Array)
-          parameters['recipients'] = parameters['recipients'].map do |recipient_data|
+          recipient_types = RECIPIENT_TYPES
+          if parameters.key?('recipient_types')
+            recipient_types = parameters['recipient_types']
+            parameters.delete('recipient_types')
+            recipient_types = [recipient_types] unless recipient_types.is_a?(Array)
+          end
+          parameters['recipients'].map! do |recipient_data|
             # if just a string, assume it is the name
             if recipient_data.is_a?(String)
-              result = @api_v5.read('contacts', {q: recipient_data, context: 'packages', type: [Rest::ARRAY_PARAMS, *RECIPIENT_TYPES]})[:data]
-              raise "No matching contact for #{recipient_data}" if 0.eql?(result['total_count'])
-              raise "Multiple matching contact for #{recipient_data} : #{result['contacts'].map{|i|i['name']}.join(', ')}" unless 1.eql?(result['total_count'])
-              matched = result['contacts'].first
+              matched = @api_v5.lookup_by_name('contacts', recipient_data, {context: 'packages', type: Rest.array_params(recipient_types)})
               recipient_data = {
                 name:           matched['name'],
                 recipient_type: matched['type']
@@ -139,44 +197,60 @@ module Aspera
         end
 
         # wait for package status to be in provided list
-        def wait_package_status(id, status_list=PACKAGE_TERMINATED)
-          parameters = options.get_option(:value)
-          spinner = nil
-          progress = nil
-          while true
+        def wait_package_status(id, status_list: PACKAGE_TERMINATED)
+          total_sent = false
+          loop do
             status = @api_v5.read("packages/#{id}/upload_details")[:data]
+            status['id'] = id
             # user asked to not follow
-            break unless parameters
+            return status if status_list.nil?
             if status['upload_status'].eql?('submitted')
-              if spinner.nil?
-                spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
-                spinner.start
-              end
-              spinner.update(title: status['upload_status'])
-              spinner.spin
-            elsif progress.nil?
-              progress = ProgressBar.create(
-                format:     '%a %B %p%% %r Mbps %e',
-                rate_scale: lambda{|rate|rate / Environment::BYTES_PER_MEBIBIT},
-                title:      'progress',
-                total:      status['bytes_total'].to_i)
+              config.progress_bar&.event(session_id: nil, type: :pre_start, info: status['upload_status'])
+            elsif !total_sent
+              config.progress_bar&.event(session_id: id, type: :session_start)
+              config.progress_bar&.event(session_id: id, type: :session_size, info: status['bytes_total'].to_i)
+              total_sent = true
             else
-              progress.progress = status['bytes_written'].to_i
+              config.progress_bar&.event(session_id: id, type: :transfer, info: status['bytes_written'].to_i)
+            end
+            if status_list.include?(status['upload_status'])
+              # if status['upload_status'].eql?('completed')
+              config.progress_bar&.event(session_id: id, type: :end)
+              return status
+              # end
             end
-            break if status_list.include?(status['upload_status'])
+            sleep(1.0)
+          end
+        end
+
+        def wait_for_job(job_id)
+          spinner = nil
+          loop do
+            status = @api_v5.read("jobs/#{job_id}", {type: :formatted})[:data]
+            return status unless JOB_RUNNING.include?(status['status'])
+            if spinner.nil?
+              spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
+              spinner.start
+            end
+            spinner.update(title: status['status'])
+            spinner.spin
             sleep(0.5)
           end
-          status['id'] = id
-          return status
+          raise 'internal error'
         end
 
-        # get a list of all entities of a given type
-        # @param entity_type [String] the type of entity to list
-        # @param query [Hash] additional query parameters
-        # @param prefix [String] optional prefix to add to the path (nil or empty string: no prefix)
-        def list_entities(entity_type, query: {}, prefix: nil)
-          path = entity_type
-          path = "#{prefix}/#{path}" unless prefix.nil? || prefix.empty?
+        # get a (full or partial) list of all entities of a given type
+        # @param type [String] the type of entity to list (just a name)
+        # @param query [Hash,nil] additional query parameters
+        # @param path [String] optional prefix to add to the path (nil or empty string: no prefix)
+        # @param item_list_key [String] key in the result to get the list of items
+        def list_entities(type:, path: nil, query: nil, item_list_key: nil)
+          query = {} if query.nil?
+          type = type.to_s if type.is_a?(Symbol)
+          item_list_key = type if item_list_key.nil?
+          raise "internal error: Invalid type #{type.class}" unless type.is_a?(String)
+          full_path = type
+          full_path = "#{path}/#{full_path}" unless path.nil? || path.empty?
           result = []
           offset = 0
           max_items = query.delete(MAX_ITEMS)
@@ -185,45 +259,198 @@ module Aspera
           query = {'limit'=> 100}.merge(query)
           loop do
             query['offset'] = offset
-            page = @api_v5.read(path, query)[:data]
-            result.concat(page[entity_type])
+            page_result = @api_v5.read(full_path, query)[:data]
+            result.concat(page_result[item_list_key])
             # reach the limit set by user ?
             if !max_items.nil? && (result.length >= max_items)
               result = result.slice(0, max_items)
               break
             end
-            break if result.length >= page['total_count']
+            break if result.length >= page_result['total_count']
             remain_pages -= 1 unless remain_pages.nil?
             break if remain_pages == 0
-            offset += page[entity_type].length
+            offset += page_result[item_list_key].length
           end
           return result
         end
 
         # lookup an entity id from its name
-        def lookup_name_to_id(entity_type, name)
-          found = list_entities(entity_type, query: {'q'=> name}).select{|i|i['name'].eql?(name)}
+        def lookup_entity_by_field(type:, value:, field: 'name', query: :default, path: nil, item_list_key: nil)
+          query = {'q'=> value} if query.eql?(:default)
+          found = list_entities(type: type, path: path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
           case found.length
-          when 0 then raise "No #{entity_type} with name = #{name}"
-          when 1 then return found.first['id']
-          else raise "Multiple #{entity_type} with name = #{name}"
+          when 0 then raise "No #{type} with #{field} = #{value}"
+          when 1 then return found.first
+          else raise "Found #{found.length} #{path} with #{field} = #{value}"
           end
         end
 
-        # translate box name to API prefix (with ending slash)
-        def box_to_prefix(box)
-          return \
+        # list all packages with optional filter
+        def list_packages_with_filter
+          filter = options.get_next_argument('filter', mandatory: false, type: Proc, default: ->(_x){true})
+          # translate box name to API prefix (with ending slash)
+          box = options.get_option(:box)
+          api_path =
+            case box
+            when ExtendedValue::ALL then '' # only admin can list all packages globally
+            when *API_LIST_MAILBOX_TYPES then box
+            else
+              group_type = options.get_option(:group_type)
+              "#{group_type}/#{lookup_entity_by_field(type: group_type, value: box)['id']}"
+            end
+          return list_entities(
+            type: 'packages',
+            query:  query_read_delete(default: {}),
+            path: api_path).select(&filter)
+        end
+
+        def package_receive(package_ids)
+          # prepare persistency if needed
+          skip_ids_persistency = nil
+          if options.get_option(:once_only, mandatory: true)
+            # read ids from persistency
+            skip_ids_persistency = PersistencyActionOnce.new(
+              manager: @agents[:persistency],
+              data:    [],
+              id:      IdGenerator.from_list([
+                'faspex_recv',
+                options.get_option(:url, mandatory: true),
+                options.get_option(:username, mandatory: true)]))
+          end
+          case package_ids
+          when PACKAGE_ALL_INIT
+            raise 'Only with option once_only' unless skip_ids_persistency
+            skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
+            skip_ids_persistency.save
+            return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
+          when ExtendedValue::ALL
+            # TODO: if packages have same name, they will overwrite ?
+            package_ids = list_packages_with_filter.map{|p|p['id']}
+            Log.log.debug{Log.dump(:package_ids, package_ids)}
+            Log.log.debug{Log.dump(:skip_ids, skip_ids_persistency.data)}
+            package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
+            Log.log.debug{Log.dump(:package_ids, package_ids)}
+          end
+          # a single id was provided
+          # TODO: check package_ids is a list of strings
+          package_ids = [package_ids] if package_ids.is_a?(String)
+          result_transfer = []
+          param_file_list = {}
+          begin
+            param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
+          rescue Cli::BadArgument
+            # paths is optional
+          end
+          download_params = {
+            type:          'received',
+            transfer_type: TRANSFER_CONNECT
+          }
+          box = options.get_option(:box)
           case box
-          when VAL_ALL then ''
-          when *API_MAILBOXES then box
-          else "shared_inboxes/#{lookup_name_to_id('shared_inboxes', box)}"
+          when /outbox/ then download_params[:type] = 'sent'
+          when *API_LIST_MAILBOX_TYPES then nil # nothing to do
+          else # shared inbox / workgroup
+            download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
           end
+          package_ids.each do |pkg_id|
+            formatter.display_status("Receiving package #{pkg_id}")
+            # TODO: allow from sent as well ?
+            transfer_spec = @api_v5.call(
+              operation:   'POST',
+              subpath:     "packages/#{pkg_id}/transfer_spec/download",
+              headers:     {'Accept' => 'application/json'},
+              url_params:  download_params,
+              json_params: param_file_list
+            )[:data]
+            # delete flag for Connect Client
+            transfer_spec.delete('authentication')
+            statuses = transfer.start(transfer_spec)
+            result_transfer.push({'package' => pkg_id, Main::STATUS_FIELD => statuses})
+            # skip only if all sessions completed
+            if TransferAgent.session_status(statuses).eql?(:success) && skip_ids_persistency
+              skip_ids_persistency.data.push(pkg_id)
+              skip_ids_persistency.save
+            end
+          end
+          return Main.result_transfer_multiple(result_transfer)
         end
 
-        # list all packages with optional filter
-        def list_packages
-          parameters = options.get_option(:value) || {}
-          return list_entities('packages', query: parameters, prefix: box_to_prefix(options.get_option(:box)))
+        def package_action
+          command = options.get_next_command(%i[show browse status delete receive send list])
+          package_id =
+            if %i[receive show browse status delete].include?(command)
+              @pub_link_context&.key?('package_id') ? @pub_link_context['package_id'] : instance_identifier
+            end
+          case command
+          when :show
+            return {type: :single_object, data: @api_v5.read("packages/#{package_id}")[:data]}
+          when :browse
+            path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
+            # TODO: support multi-page listing ?
+            params = {
+              # recipient_user_id: 25,
+              # offset:            0,
+              # limit:             25
+            }
+            result = @api_v5.call({
+              operation:   'POST',
+              subpath:     "packages/#{package_id}/files/received",
+              headers:     {'Accept' => 'application/json'},
+              url_params:  params,
+              json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
+            formatter.display_item_count(result['item_count'], result['total_count'])
+            return {type: :object_list, data: result['items']}
+          when :status
+            status = wait_package_status(package_id, status_list: nil)
+            return {type: :single_object, data: status}
+          when :delete
+            ids = package_id
+            ids = [ids] unless ids.is_a?(Array)
+            raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
+            # API returns 204, empty on success
+            @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
+            return Main.result_status('Package(s) deleted')
+          when :receive
+            return package_receive(package_id)
+          when :send
+            parameters = value_create_modify(command: command)
+            normalize_recipients(parameters)
+            package = @api_v5.create('packages', parameters)[:data]
+            shared_folder = options.get_option(:shared_folder)
+            if shared_folder.nil?
+              # send from local files
+              transfer_spec = @api_v5.call(
+                operation:   'POST',
+                subpath:     "packages/#{package['id']}/transfer_spec/upload",
+                headers:     {'Accept' => 'application/json'},
+                url_params:  {transfer_type: TRANSFER_CONNECT},
+                json_params: {paths: transfer.source_list}
+              )[:data]
+              # well, we asked a TS for connect, but we actually want a generic one
+              transfer_spec.delete('authentication')
+              return Main.result_transfer(transfer.start(transfer_spec))
+            else
+              # send from remote shared folder
+              if (m = shared_folder.match(REGEX_LOOKUP_ID_BY_FIELD))
+                shared_folder = lookup_entity_by_field(type: 'shared_folders', value: m[2])['id']
+              end
+              transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
+              # start remote transfer and get first status
+              result = @api_v5.create("packages/#{package['id']}/remote_transfer", transfer_request)[:data]
+              result['id'] = package['id']
+              unless result['status'].eql?('completed')
+                formatter.display_status("Package #{package['id']}")
+                result = wait_package_status(package['id'])
+              end
+              return {type: :single_object, data: result}
+            end
+          when :list
+            return {
+              type:   :object_list,
+              data:   list_packages_with_filter,
+              fields: %w[id title release_date total_bytes total_files created_time state]
+            }
+          end # case package
         end
 
         ACTIONS = %i[health version user bearer_token packages shared_folders admin gateway postprocessing].freeze
@@ -259,148 +486,19 @@ module Aspera
           when :bearer_token
             return {type: :text, data: @api_v5.oauth_token}
           when :packages
-            command = options.get_next_command(%i[list show browse status delete send receive])
-            case command
-            when :list
-              return {
-                type:   :object_list,
-                data:   list_packages,
-                fields: %w[id title release_date total_bytes total_files created_time state]
-              }
-            when :show
-              id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-              id ||= instance_identifier
-              return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
-            when :browse
-              id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-              id ||= instance_identifier
-              path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
-              # TODO: support multi-page listing ?
-              params = {
-                # recipient_user_id: 25,
-                # offset:            0,
-                # limit:             25
-              }
-              result = @api_v5.call({
-                operation:   'POST',
-                subpath:     "packages/#{id}/files/received",
-                headers:     {'Accept' => 'application/json'},
-                url_params:  params,
-                json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
-              formatter.display_item_count(result['item_count'], result['total_count'])
-              return {type: :object_list, data: result['items']}
-            when :status
-              status = wait_package_status(instance_identifier)
-              return {type: :single_object, data: status}
-            when :delete
-              ids = instance_identifier
-              ids = [ids] unless ids.is_a?(Array)
-              raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
-              # API returns 204, empty on success
-              @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
-              return Main.result_status('Package(s) deleted')
-            when :send
-              parameters = options.get_option(:value, is_type: :mandatory)
-              raise CliBadArgument, 'Value must be Hash, refer to API' unless parameters.is_a?(Hash)
-              normalize_recipients(parameters)
-              package = @api_v5.create('packages', parameters)[:data]
-              shared_folder = options.get_option(:shared_folder)
-              if shared_folder.nil?
-                # TODO: option to send from remote source or httpgw
-                transfer_spec = @api_v5.call(
-                  operation:   'POST',
-                  subpath:     "packages/#{package['id']}/transfer_spec/upload",
-                  headers:     {'Accept' => 'application/json'},
-                  url_params:  {transfer_type: TRANSFER_CONNECT},
-                  json_params: {paths: transfer.source_list}
-                )[:data]
-                # well, we asked a TS for connect, but we actually want a generic one
-                transfer_spec.delete('authentication')
-                return Main.result_transfer(transfer.start(transfer_spec))
-              else
-                if !shared_folder.to_i.to_s.eql?(shared_folder)
-                  shared_folder = lookup_name_to_id('shared_folders', shared_folder)
-                end
-                transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
-                # start remote transfer and get first status
-                result = @api_v5.create("packages/#{package['id']}/remote_transfer", transfer_request)[:data]
-                result['id'] = package['id']
-                unless result['status'].eql?('completed')
-                  formatter.display_status("Package #{package['id']}")
-                  result = wait_package_status(package['id'])
-                end
-                return {type: :single_object, data: result}
-              end
-            when :receive
-              # prepare persistency if needed
-              skip_ids_persistency = nil
-              if options.get_option(:once_only, is_type: :mandatory)
-                # read ids from persistency
-                skip_ids_persistency = PersistencyActionOnce.new(
-                  manager: @agents[:persistency],
-                  data:    [],
-                  id:      IdGenerator.from_list([
-                    'faspex_recv',
-                    options.get_option(:url, is_type: :mandatory),
-                    options.get_option(:username, is_type: :mandatory),
-                    PACKAGE_TYPE_RECEIVED]))
-              end
-              # one or several packages
-              package_ids = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-              package_ids ||= instance_identifier
-              case package_ids
-              when PACKAGE_ALL_INIT
-                raise 'Only with option once_only' unless skip_ids_persistency
-                skip_ids_persistency.data.clear.concat(list_packages.map{|p|p['id']})
-                skip_ids_persistency.save
-                return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
-              when VAL_ALL
-                # TODO: if packages have same name, they will overwrite ?
-                package_ids = list_packages.map{|p|p['id']}
-                Log.dump(:package_ids, package_ids)
-                Log.dump(:package_ids, skip_ids_persistency.data)
-                package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
-                Log.dump(:package_ids, package_ids)
-              end
-              # a single id was provided
-              # TODO: check package_ids is a list of strings
-              package_ids = [package_ids] if package_ids.is_a?(String)
-              result_transfer = []
-              package_ids.each do |pkg_id|
-                formatter.display_status("Receiving package #{pkg_id}")
-                param_file_list = {}
-                begin
-                  param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
-                rescue Aspera::Cli::CliBadArgument
-                  # paths is optional
-                end
-                # TODO: allow from sent as well ?
-                transfer_spec = @api_v5.call(
-                  operation:   'POST',
-                  subpath:     "packages/#{pkg_id}/transfer_spec/download",
-                  headers:     {'Accept' => 'application/json'},
-                  url_params:  {transfer_type: TRANSFER_CONNECT, type: PACKAGE_TYPE_RECEIVED},
-                  json_params: param_file_list
-                )[:data]
-                # delete flag for Connect Client
-                transfer_spec.delete('authentication')
-                statuses = transfer.start(transfer_spec)
-                result_transfer.push({'package' => pkg_id, Main::STATUS_FIELD => statuses})
-                # skip only if all sessions completed
-                if TransferAgent.session_status(statuses).eql?(:success) && skip_ids_persistency
-                  skip_ids_persistency.data.push(pkg_id)
-                  skip_ids_persistency.save
-                end
-              end
-              return Main.result_transfer_multiple(result_transfer)
-            end # case package
+            return package_action
           when :shared_folders
             all_shared_folders = @api_v5.read('shared_folders')[:data]['shared_folders']
             case options.get_next_command(%i[list browse])
             when :list
               return {type: :object_list, data: all_shared_folders}
             when :browse
-              shared_folder_id = instance_identifier
+              shared_folder_id = instance_identifier do |field, value|
+                matches = all_shared_folders.select{|i|i[field].eql?(value)}
+                raise "no match for #{field} = #{value}" if matches.empty?
+                raise "multiple matches for #{field} = #{value}" if matches.length > 1
+                matches.first['id']
+              end
               path = options.get_next_argument('folder path', mandatory: false) || '/'
               node = all_shared_folders.find{|i|i['id'].eql?(shared_folder_id)}
               raise "No such shared folder id #{shared_folder_id}" if node.nil?
@@ -418,12 +516,14 @@ module Aspera
               end
             end
           when :admin
-            case options.get_next_command(%i[resource smtp])
+            case options.get_next_command(%i[resource smtp].freeze)
             when :resource
-              res_type = options.get_next_command(%i[accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs metadata_profiles
-                                                     email_notifications])
+              res_type = options.get_next_command(ADMIN_RESOURCES)
               res_path = list_key = res_type.to_s
               id_as_arg = false
+              display_fields = nil
+              adm_api = @api_v5
+              available_commands = [].concat(Plugin::ALL_OPS)
               case res_type
               when :metadata_profiles
                 res_path = 'configuration/metadata_profiles'
@@ -431,50 +531,96 @@ module Aspera
               when :email_notifications
                 list_key = false
                 id_as_arg = 'type'
+              when :accounts
+                display_fields = Formatter.all_but('user_profile_data_attributes')
+              when :oauth_clients
+                display_fields = Formatter.all_but('public_key')
+                adm_api = Rest.new(@api_v5.params.merge({base_url: auth_api_url}))
+              when :shared_inboxes, :workgroups
+                available_commands.push(:members, :saml_groups, :invite_external_collaborator)
               end
-              display_fields =
-                case res_type
-                when :accounts then [:all_but, 'user_profile_data_attributes']
-                when :oauth_clients then [:all_but, 'public_key']
+              res_command = options.get_next_command(available_commands)
+              case res_command
+              when *Plugin::ALL_OPS
+                return entity_command(res_command, adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
+              when :invite_external_collaborator
+                shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
+                creation_payload = value_create_modify(command: res_command, type: [Hash, String])
+                creation_payload = {'email_address' => creation_payload} if creation_payload.is_a?(String)
+                res_path = "#{res_type}/#{shared_inbox_id}/external_collaborator"
+                result = adm_api.create(res_path, creation_payload)[:data]
+                formatter.display_status(result['message'])
+                result = lookup_entity_by_field(type: 'members', path: "#{res_type}/#{shared_inbox_id}", value: creation_payload['email_address'], query: {})
+                return {type: :single_object, data: result}
+              when :members, :saml_groups
+                res_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
+                res_prefix = "#{res_type}/#{res_id}"
+                res_path = "#{res_prefix}/#{res_command}"
+                list_key = res_command.to_s
+                list_key = 'groups' if res_command.eql?(:saml_groups)
+                sub_command = options.get_next_command(%i[create list modify delete])
+                if sub_command.eql?(:create) && options.get_option(:value).nil?
+                  raise "use option 'value' to provide saml group_id and access (refer to API)" unless res_command.eql?(:members)
+                  # first arg is one user name or list of users
+                  users = options.get_next_argument('user id, or email, or list of')
+                  users = [users] unless users.is_a?(Array)
+                  users = users.map do |user|
+                    if (m = user.match(REGEX_LOOKUP_ID_BY_FIELD))
+                      lookup_entity_by_field(
+                        type: 'accounts', field: m[1], value: m[2],
+                        query: {type: Rest.array_params(%w{local_user saml_user self_registered_user external_user})})['id']
+                    else
+                      # it's the user id (not member id...)
+                      user
+                    end
+                  end
+                  access = options.get_next_argument('level', mandatory: false, expected: %i[submit_only standard shared_inbox_admin], default: :standard)
+                  # TODO: unshift to command line parameters instead of using deprecated option "value"
+                  options.set_option(:value, {user: users.map{|u|{id: u, access: access}}})
                 end
-              adm_api = @api_v5
-              if res_type.eql?(:oauth_clients)
-                adm_api = Rest.new(@api_v5.params.merge({base_url: @faspex5_api_auth_url}))
+                return entity_command(sub_command, adm_api, res_path, item_list_key: list_key) do |field, value|
+                         lookup_entity_by_field(
+                           type: 'accounts', field: field, value: value,
+                           query: {type: Rest.array_params(%w{local_user saml_user self_registered_user external_user})})['id']
+                       end
               end
-              return entity_action(adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
             when :smtp
               smtp_path = 'configuration/smtp'
-              case options.get_next_command(%i[show create modify delete test])
+              smtp_cmd = options.get_next_command(%i[show create modify delete test])
+              case smtp_cmd
               when :show
                 return { type: :single_object, data: @api_v5.read(smtp_path)[:data] }
               when :create
-                return { type: :single_object, data: @api_v5.create(smtp_path, options.get_option(:value, is_type: :mandatory))[:data] }
+                return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd))[:data] }
               when :modify
-                return { type: :single_object, data: @api_v5.modify(smtp_path, options.get_option(:value, is_type: :mandatory))[:data] }
+                return { type: :single_object, data: @api_v5.update(smtp_path, value_create_modify(command: smtp_cmd))[:data] }
               when :delete
                 return { type: :single_object, data: @api_v5.delete(smtp_path)[:data] }
               when :test
                 test_data = options.get_next_argument('Email or test data, see API')
                 test_data = {test_email_recipient: test_data} if test_data.is_a?(String)
-                return { type: :single_object, data: @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data] }
+                creation = @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data]
+                result = wait_for_job(creation['job_id'])
+                result['serialized_args'] = JSON.parse(result['serialized_args']) rescue result['serialized_args']
+                return { type: :single_object, data: result }
               end
             end
           when :gateway
             require 'aspera/faspex_gw'
-            url = options.get_option(:value, is_type: :mandatory)
+            url = value_create_modify(command: command, type: String)
             uri = URI.parse(url)
             server = WebServerSimple.new(uri)
             server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
+            # on ctrl-c, tell server main loop to exit
             trap('INT') { server.shutdown }
-            formatter.display_status("Faspex 4 gateway listening on #{url}")
+            formatter.display_status("Gateway for Faspex 4-style API listening on #{url}")
             Log.log.info("Listening on #{url}")
             # this is blocking until server exits
             server.start
             return Main.result_status('Gateway terminated')
           when :postprocessing
-            require 'aspera/faspex_postproc'
-            parameters = options.get_option(:value, is_type: :mandatory)
-            raise 'parameters must be Hash' unless parameters.is_a?(Hash)
+            require 'aspera/faspex_postproc' # cspell:disable-line
+            parameters = value_create_modify(command: command)
             parameters = parameters.symbolize_keys
             raise 'Missing key: url' unless parameters.key?(:url)
             uri = URI.parse(parameters[:url])
@@ -482,8 +628,9 @@ module Aspera
             parameters[:processing][:root] = uri.path
             server = WebServerSimple.new(uri, certificate: parameters[:certificate])
             server.mount(uri.path, Faspex4PostProcServlet, parameters[:processing])
+            # on ctrl-c, tell server main loop to exit
             trap('INT') { server.shutdown }
-            formatter.display_status("Faspex 4 post processing listening on #{uri.port}")
+            formatter.display_status("Web-hook for Faspex 4-style post processing listening on #{uri.port}")
             Log.log.info("Listening on #{uri.port}")
             # this is blocking until server exits
             server.start