aspera-cli 4.16.0 → 4.17.0

data/lib/aspera/cli/plugins/faspex5.rb

@@ -15,10 +15,9 @@ require 'tty-spinner'
 module Aspera
   module Cli
     module Plugins
-      class Faspex5 < Aspera::Cli::BasicAuthPlugin
+      class Faspex5 < Cli::BasicAuthPlugin
         RECIPIENT_TYPES = %w[user workgroup external_user distribution_list shared_inbox].freeze
         PACKAGE_TERMINATED = %w[completed failed].freeze
-        API_DETECT = 'api/v5/configuration/ping'
         # list of supported mailbox types (to list packages)
         API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
         PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
@@ -28,34 +27,49 @@ module Aspera
           accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
           metadata_profiles email_notifications alternate_addresses
         ].freeze
+        # states for jobs not in final state
         JOB_RUNNING = %w[queued working].freeze
-        STANDARD_PATH = '/aspera/faspex'
+        PATH_STANDARD_ROOT = '/aspera/faspex'
+        PATH_API_V5 = 'api/v5'
+        # endpoint for authentication API
+        PATH_AUTH = 'auth'
+        PATH_HEALTH = 'configuration/ping'
         PER_PAGE_DEFAULT = 100
-        private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE PER_PAGE_DEFAULT])
+        # OAuth methods supported
+        STD_AUTH_TYPES = %i[web jwt boot].freeze
+        HEADER_ITERATION_TOKEN = 'X-Aspera-Next-Iteration-Token'
+        private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED PATH_HEALTH API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE PER_PAGE_DEFAULT
+                             STD_AUTH_TYPES])
         class << self
           def application_name
             'Faspex'
           end
 
           def detect(address_or_url)
+            # add scheme if missing
             address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
             urls = [address_or_url]
-            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
-
+            urls.push("#{address_or_url}#{PATH_STANDARD_ROOT}") unless address_or_url.end_with?(PATH_STANDARD_ROOT)
+            error = nil
             urls.each do |base_url|
+              # Faspex is always HTTPS
               next unless base_url.start_with?('https://')
               api = Rest.new(base_url: base_url, redirect_max: 1)
-              result = api.read(API_DETECT)
+              path_api_detect = "#{PATH_API_V5}/#{PATH_HEALTH}"
+              result = api.read(path_api_detect)
               next unless result[:http].code.start_with?('2') && result[:http].body.strip.empty?
-              url_length = -2 - API_DETECT.length
+              # end is at -1, and substract 1 for "/"
+              url_length = -2 - path_api_detect.length
               # take redirect if any
               return {
                 version: result[:http]['x-ibm-aspera'] || '5',
                 url:     result[:http].uri.to_s[0..url_length]
               }
             rescue StandardError => e
+              error = e
               Log.log.debug{"detect error: #{e}"}
             end
+            raise error if error
             return nil
           end
 
@@ -91,31 +105,25 @@ module Aspera
             }
           end
 
+          # @return true if the URL is a public link
           def public_link?(url)
-            url.include?('/public/')
+            url.include?('?context=')
           end
         end
 
-        def initialize(env)
-          super(env)
+        def initialize(**env)
+          super
           options.declare(:client_id, 'OAuth client identifier')
           options.declare(:client_secret, 'OAuth client secret')
           options.declare(:redirect_uri, 'OAuth redirect URI for web authentication')
-          options.declare(:auth, 'OAuth type of authentication', values: %i[boot].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
+          options.declare(:auth, 'OAuth type of authentication', values: STD_AUTH_TYPES, default: :jwt)
           options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
           options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
           options.declare(:box, "Package inbox, either shared inbox name or one of: #{API_LIST_MAILBOX_TYPES.join(', ')} or #{ExtendedValue::ALL}", default: 'inbox')
           options.declare(:shared_folder, 'Send package with files from shared folder')
           options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
           options.parse_options!
-        end
-
-        def api_url
-          return "#{@faspex5_api_base_url}/api/v5"
-        end
-
-        def auth_api_url
-          return "#{@faspex5_api_base_url}/auth"
+          @pub_link_context = nil
         end
 
         def set_api
@@ -124,60 +132,63 @@ module Aspera
           auth_type = self.class.public_link?(@faspex5_api_base_url) ? :public_link : options.get_option(:auth, mandatory: true)
           case auth_type
           when :public_link
+            # resolve any redirect
+            @faspex5_api_base_url = Rest.new(base_url: @faspex5_api_base_url, redirect_max: 3).read('')[:http].uri.to_s
             encoded_context = Rest.decode_query(URI.parse(@faspex5_api_base_url).query)['context']
             raise 'Bad faspex5 public link, missing context in query' if encoded_context.nil?
+            # public link information (allowed usage)
             @pub_link_context = JSON.parse(Base64.decode64(encoded_context))
             Log.log.trace1{Log.dump(:@pub_link_context, @pub_link_context)}
             # ok, we have the additional parameters, get the base url
             @faspex5_api_base_url = @faspex5_api_base_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
-            @api_v5 = Rest.new({
-              base_url: api_url,
+            @api_v5 = Rest.new(
+              base_url: "#{@faspex5_api_base_url}/#{PATH_API_V5}",
               headers:  {'Passcode' => @pub_link_context['passcode']}
-            })
+            )
           when :boot
             # the password here is the token copied directly from browser in developer mode
-            @api_v5 = Rest.new({
-              base_url: api_url,
+            @api_v5 = Rest.new(
+              base_url: "#{@faspex5_api_base_url}/#{PATH_API_V5}",
               headers:  {'Authorization' => options.get_option(:password, mandatory: true)}
-            })
+            )
           when :web
             # opens a browser and ask user to auth using web
-            @api_v5 = Rest.new({
-              base_url: api_url,
+            @api_v5 = Rest.new(
+              base_url: "#{@faspex5_api_base_url}/#{PATH_API_V5}",
               auth:     {
                 type:         :oauth2,
-                base_url:     auth_api_url,
+                base_url:     "#{@faspex5_api_base_url}/#{PATH_AUTH}",
                 grant_method: :web,
                 client_id:    options.get_option(:client_id, mandatory: true),
-                web:          {redirect_uri: options.get_option(:redirect_uri, mandatory: true)}
-              }})
+                redirect_uri: options.get_option(:redirect_uri, mandatory: true)
+              })
           when :jwt
             app_client_id = options.get_option(:client_id, mandatory: true)
-            @api_v5 = Rest.new({
-              base_url: api_url,
+            @api_v5 = Rest.new(
+              base_url: "#{@faspex5_api_base_url}/#{PATH_API_V5}",
               auth:     {
-                type:         :oauth2,
-                base_url:     auth_api_url,
-                grant_method: :jwt,
-                client_id:    app_client_id,
-                jwt:          {
-                  payload:         {
-                    iss: app_client_id,    # issuer
-                    aud: app_client_id,    # audience (this field is not clear...)
-                    sub: "user:#{options.get_option(:username, mandatory: true)}" # subject is a user
-                  },
-                  private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, mandatory: true), options.get_option(:passphrase)),
-                  headers:         {typ: 'JWT'}
-                }
-              }})
-          else error_unexpected_value(auth_type)
+                type:            :oauth2,
+                grant_method:    :jwt,
+                base_url:        "#{@faspex5_api_base_url}/#{PATH_AUTH}",
+                client_id:       app_client_id,
+                payload:         {
+                  iss: app_client_id, # issuer
+                  aud: app_client_id, # audience (this field is not clear...)
+                  sub: "user:#{options.get_option(:username, mandatory: true)}" # subject is a user
+                },
+                private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, mandatory: true), options.get_option(:passphrase)),
+                headers:         {typ: 'JWT'}
+              })
+          else Aspera.error_unexpected_value(auth_type)
           end
+          # in case user wants to use HTTPGW tell transfer agent how to get address
+          transfer.httpgw_url_cb = lambda { @api_v5.read('account')[:data]['gateway_url'] }
         end
 
         # if recipient is just an email, then convert to expected API hash : name and type
         def normalize_recipients(parameters)
           return unless parameters.key?('recipients')
-          assert_type(parameters['recipients'], Array){'recipients'}
+          Aspera.assert_type(parameters['recipients'], Array){'recipients'}
           recipient_types = RECIPIENT_TYPES
           if parameters.key?('recipient_types')
             recipient_types = parameters['recipient_types']
@@ -185,7 +196,7 @@ module Aspera
             recipient_types = [recipient_types] unless recipient_types.is_a?(Array)
           end
           parameters['recipients'].map! do |recipient_data|
-            # if just a string, assume it is the name
+            # if just a string, make a general lookup and build expected name/type hash
             if recipient_data.is_a?(String)
               matched = @api_v5.lookup_by_name('contacts', recipient_data, {context: 'packages', type: Rest.array_params(recipient_types)})
               recipient_data = {
@@ -238,7 +249,7 @@ module Aspera
             spinner.spin
             sleep(0.5)
           end
-          error_unreachable_line
+          Aspera.error_unreachable_line
         end
 
         # Get a (full or partial) list of all entities of a given type
@@ -248,7 +259,7 @@ module Aspera
         # @param item_list_key [String] key in the result to get the list of items
         def list_entities(type:, real_path: nil, query: {}, item_list_key: nil)
           type = type.to_s if type.is_a?(Symbol)
-          assert_type(type, String)
+          Aspera.assert_type(type, String)
           item_list_key = type if item_list_key.nil?
           full_path = real_path.nil? ? type : real_path
           result = []
@@ -277,7 +288,7 @@ module Aspera
         # lookup an entity id from its name
         def lookup_entity_by_field(type:, value:, field: 'name', query: :default, real_path: nil, item_list_key: nil)
           if query.eql?(:default)
-            assert(field.eql?('name')){'Default query is on name only'}
+            Aspera.assert(field.eql?('name')){'Default query is on name only'}
             query = {'q'=> value}
           end
           found = list_entities(type: type, real_path: real_path, query: query, item_list_key: item_list_key).select{|i|i[field].eql?(value)}
@@ -313,7 +324,7 @@ module Aspera
           if options.get_option(:once_only, mandatory: true)
             # read ids from persistency
             skip_ids_persistency = PersistencyActionOnce.new(
-              manager: @agents[:persistency],
+              manager: persistency,
               data:    [],
               id:      IdGenerator.from_list([
                 'faspex_recv',
@@ -325,7 +336,7 @@ module Aspera
           packages = []
           case package_ids
           when ExtendedValue::INIT
-            assert(skip_ids_persistency){'Only with option once_only'}
+            Aspera.assert(skip_ids_persistency){'Only with option once_only'}
             skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
             skip_ids_persistency.save
             return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
@@ -339,8 +350,8 @@ module Aspera
           else
             # a single id was provided, or a list of ids
             package_ids = [package_ids] unless package_ids.is_a?(Array)
-            assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
-            assert(package_ids.all?(String)){'Package id shall be String'}
+            Aspera.assert_type(package_ids, Array){'Expecting a single package id or a list of ids'}
+            Aspera.assert(package_ids.all?(String)){'Package id shall be String'}
             # packages = package_ids.map{|pkg_id|@api_v5.read("packages/#{pkg_id}")[:data]}
             packages = package_ids.map{|pkg_id|{'id'=>pkg_id}}
           end
@@ -386,6 +397,45 @@ module Aspera
           return Main.result_transfer_multiple(result_transfer)
         end
 
+        # browse a folder
+        # @param browse_endpoint [String] the endpoint to browse
+        def browse_folder(browse_endpoint)
+          path = options.get_next_argument('folder path', mandatory: false, default: '/')
+          query = query_read_delete(default: {})
+          query['filters'] = {} unless query.key?('filters')
+          filters = query.delete('filters')
+          filters['basenames'] = [] unless filters.key?('basenames')
+          Aspera.assert_type(filters, Hash){'filters'}
+          max_items = query.delete('max')
+          recursive = query.delete('recursive')
+          all_items = []
+          folders_to_process = [path]
+          until folders_to_process.empty?
+            path = folders_to_process.shift
+            query.delete('iteration_token')
+            loop do
+              response = @api_v5.call(
+                operation:   'POST',
+                subpath:     browse_endpoint,
+                headers:     {'Accept' => 'application/json', 'Content-Type' => 'application/json'},
+                url_params:  query,
+                json_params: {'path' => path, 'filters' => filters})
+              all_items.concat(response[:data]['items'])
+              if recursive
+                folders_to_process.concat(response[:data]['items'].select{|i|i['type'].eql?('directory')}.map{|i|i['path']})
+              end
+              if !max_items.nil? && (all_items.count >= max_items)
+                all_items = all_items.slice(0, max_items) if all_items.count > max_items
+                break
+              end
+              iteration_token = response[:http][HEADER_ITERATION_TOKEN]
+              break if iteration_token.nil? || iteration_token.empty?
+              query['iteration_token'] = iteration_token
+            end
+          end
+          return {type: :object_list, data: all_items}
+        end
+
         def package_action
           command = options.get_next_command(%i[show browse status delete receive send list])
           package_id =
@@ -396,36 +446,34 @@ module Aspera
           when :show
             return {type: :single_object, data: @api_v5.read("packages/#{package_id}")[:data]}
           when :browse
-            path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
-            # TODO: support multi-page listing ?
-            params = {
-              # recipient_user_id: 25,
-              # offset:            0,
-              # limit:             25
-            }
-            result = @api_v5.call({
-              operation:   'POST',
-              subpath:     "packages/#{package_id}/files/received",
-              headers:     {'Accept' => 'application/json'},
-              url_params:  params,
-              json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
-            formatter.display_item_count(result['item_count'], result['total_count'])
-            return {type: :object_list, data: result['items']}
+            location = case options.get_option(:box)
+            when 'inbox' then 'received'
+            when 'outbox' then 'sent'
+            else raise 'Browse only available for inbox and outbox'
+            end
+            return browse_folder("packages/#{package_id}/files/#{location}")
           when :status
             status = wait_package_status(package_id, status_list: nil)
             return {type: :single_object, data: status}
           when :delete
             ids = package_id
             ids = [ids] unless ids.is_a?(Array)
-            assert_type(ids, Array){'Package identifier'}
-            assert(ids.all?(String)){'Package id shall be String'}
+            Aspera.assert_type(ids, Array){'Package identifier'}
+            Aspera.assert(ids.all?(String)){'Package id shall be String'}
             # API returns 204, empty on success
-            @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
+            @api_v5.call(operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids})
             return Main.result_status('Package(s) deleted')
           when :receive
             return package_receive(package_id)
           when :send
             parameters = value_create_modify(command: command)
+            # autofill recipient for public url
+            if @pub_link_context&.key?('recipient_type') && !parameters.key?('recipients')
+              parameters['recipients'] = [{
+                name:           @pub_link_context['name'],
+                recipient_type: @pub_link_context['recipient_type']
+              }]
+            end
             normalize_recipients(parameters)
             package = @api_v5.create('packages', parameters)[:data]
             shared_folder = options.get_option(:shared_folder)
@@ -465,7 +513,7 @@ module Aspera
           end # case package
         end
 
-        ACTIONS = %i[health version user bearer_token packages shared_folders admin gateway postprocessing].freeze
+        ACTIONS = %i[health version user bearer_token packages shared_folders admin gateway postprocessing invitations].freeze
 
         def execute_action
           command = options.get_next_command(ACTIONS)
@@ -485,7 +533,9 @@ module Aspera
             end
             return nagios.result
           when :user
-            case options.get_next_command(%i[profile])
+            case options.get_next_command(%i[account profile])
+            when :account
+              return { type: :single_object, data: @api_v5.read('account')[:data] }
             when :profile
               case options.get_next_command(%i[show modify])
               when :show
@@ -511,21 +561,9 @@ module Aspera
                 raise "multiple matches for #{field} = #{value}" if matches.length > 1
                 matches.first['id']
               end
-              path = options.get_next_argument('folder path', mandatory: false) || '/'
               node = all_shared_folders.find{|i|i['id'].eql?(shared_folder_id)}
               raise "No such shared folder id #{shared_folder_id}" if node.nil?
-              result = @api_v5.call({
-                operation:   'POST',
-                subpath:     "nodes/#{node['node_id']}/shared_folders/#{shared_folder_id}/browse",
-                headers:     {'Accept' => 'application/json', 'Content-Type' => 'application/json'},
-                json_params: {'path': path, 'filters': {'basenames': []}},
-                url_params:  {offset: 0, limit: 100}
-              })[:data]
-              if result.key?('items')
-                return {type: :object_list, data: result['items']}
-              else
-                return {type: :single_object, data: result['self']}
-              end
+              return browse_folder("nodes/#{node['node_id']}/shared_folders/#{shared_folder_id}/browse")
             end
           when :admin
             case options.get_next_command(%i[resource smtp].freeze)
@@ -550,12 +588,12 @@ module Aspera
                 display_fields = Formatter.all_but('user_profile_data_attributes')
               when :oauth_clients
                 display_fields = Formatter.all_but('public_key')
-                adm_api = Rest.new(@api_v5.params.merge({base_url: auth_api_url}))
+                adm_api = Rest.new(**@api_v5.params.merge(base_url: "#{@faspex5_api_base_url}/#{PATH_AUTH}"))
               when :shared_inboxes, :workgroups
                 available_commands.push(:members, :saml_groups, :invite_external_collaborator)
                 special_query = {'all': true}
               when :nodes
-                available_commands.push(:shared_folders)
+                available_commands.push(:shared_folders, :browse)
               end
               res_command = options.get_next_command(available_commands)
               case res_command
@@ -573,6 +611,8 @@ module Aspera
                          lookup_entity_by_field(
                            type: 'shared_folders', real_path: sh_path, field: field, value: value)['id']
                        end
+              when :browse
+                return browse_folder("#{res_path}/#{instance_identifier}/browse")
               when :invite_external_collaborator
                 shared_inbox_id = instance_identifier { |field, value| lookup_entity_by_field(type: res_type.to_s, field: field, value: value)['id']}
                 creation_payload = value_create_modify(command: res_command, type: [Hash, String])
@@ -637,34 +677,41 @@ module Aspera
                 return { type: :single_object, data: result }
               end
             end
+          when :invitations
+            invitation_endpoint = 'invitations'
+            invitation_command = options.get_next_command(%i[resend].concat(Plugin::ALL_OPS))
+            case invitation_command
+            when :create
+              return do_bulk_operation(command: invitation_command, descr: 'data') do |params|
+                invitation_endpoint = params.key?('recipient_name') ? 'public_invitations' : 'invitations'
+                @api_v5.create(invitation_endpoint, params)[:data]
+              end
+            when :resend
+              @api_v5.create("#{invitation_endpoint}/#{instance_identifier}/resend")
+              return Main.result_status('Invitation resent')
+            else
+              return entity_command(
+                invitation_command, @api_v5, invitation_endpoint, item_list_key: invitation_endpoint,
+                display_fields: %w[id public recipient_type recipient_name email_address])
+            end
           when :gateway
             require 'aspera/faspex_gw'
             url = value_create_modify(command: command, description: 'listening url (e.g. https://localhost:12345)', type: String)
             uri = URI.parse(url)
             server = WebServerSimple.new(uri)
             server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
-            # on ctrl-c, tell server main loop to exit
-            trap('INT') { server.shutdown }
-            formatter.display_status("Gateway for Faspex 4-style API listening on #{url}")
-            Log.log.info("Listening on #{url}")
-            # this is blocking until server exits
             server.start
             return Main.result_status('Gateway terminated')
           when :postprocessing
             require 'aspera/faspex_postproc' # cspell:disable-line
             parameters = value_create_modify(command: command)
             parameters = parameters.symbolize_keys
-            assert(parameters.key?(:url)){'Missing key: url'}
+            Aspera.assert(parameters.key?(:url)){'Missing key: url'}
             uri = URI.parse(parameters[:url])
             parameters[:processing] ||= {}
             parameters[:processing][:root] = uri.path
             server = WebServerSimple.new(uri, certificate: parameters[:certificate])
             server.mount(uri.path, Faspex4PostProcServlet, parameters[:processing])
-            # on ctrl-c, tell server main loop to exit
-            trap('INT') { server.shutdown }
-            formatter.display_status("Web-hook for Faspex 4-style post processing listening on #{uri.port}")
-            Log.log.info("Listening on #{uri.port}")
-            # this is blocking until server exits
             server.start
             return Main.result_status('Gateway terminated')
           end # case command