aspera-cli 4.12.0 → 4.13.0

data/lib/aspera/cli/plugins/faspex5.rb

@@ -18,12 +18,20 @@ module Aspera
         RECIPIENT_TYPES = %w[user workgroup external_user distribution_list shared_inbox].freeze
         PACKAGE_TERMINATED = %w[completed failed].freeze
         API_DETECT = 'api/v5/configuration/ping'
+        # list of supported mailbox types
+        API_MAILBOXES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
+        PACKAGE_TYPE_RECEIVED = 'received'
+        PACKAGE_ALL_INIT = 'INIT'
+        private_constant(*%i[RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_MAILBOXES PACKAGE_TYPE_RECEIVED])
         class << self
           def detect(base_url)
             api = Rest.new(base_url: base_url, redirect_max: 1)
             result = api.read(API_DETECT)
             if result[:http].code.start_with?('2') && result[:http].body.strip.empty?
-              return {version: '5', url: result[:http].uri.to_s[0..-(API_DETECT.length + 2)]}
+              suffix_length = -2 - API_DETECT.length
+              return {
+                version: result[:http]['x-ibm-aspera'] || '5',
+                url:     result[:http].uri.to_s[0..suffix_length]}
             end
             return nil
           end
@@ -36,19 +44,39 @@ module Aspera
           options.add_opt_simple(:client_id, 'OAuth client identifier')
           options.add_opt_simple(:client_secret, 'OAuth client secret')
           options.add_opt_simple(:redirect_uri, 'OAuth redirect URI for web authentication')
-          options.add_opt_list(:auth, [:boot].concat(Oauth::STD_AUTH_TYPES), 'OAuth type of authentication')
+          options.add_opt_list(:auth, %i[boot link].concat(Oauth::STD_AUTH_TYPES), 'OAuth type of authentication')
+          options.add_opt_simple(:box, "Package inbox, either shared inbox name or one of #{API_MAILBOXES}")
           options.add_opt_simple(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
           options.add_opt_simple(:passphrase, 'RSA private key passphrase')
           options.add_opt_simple(:shared_folder, 'Shared folder source for package files')
+          options.add_opt_simple(:link, 'public link for specific operation')
           options.set_option(:auth, :jwt)
+          options.set_option(:box, 'inbox')
           options.parse_options!
         end
 
         def set_api
-          @faspex5_api_base_url = options.get_option(:url, is_type: :mandatory).gsub(%r{/+$}, '')
+          public_link = options.get_option(:link)
+          unless public_link.nil?
+            @faspex5_api_base_url = public_link.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
+            options.set_option(:auth, :link)
+          end
+          @faspex5_api_base_url ||= options.get_option(:url, is_type: :mandatory).gsub(%r{/+$}, '')
           @faspex5_api_auth_url = "#{@faspex5_api_base_url}/auth"
           faspex5_api_v5_url = "#{@faspex5_api_base_url}/api/v5"
           case options.get_option(:auth, is_type: :mandatory)
+          when :link
+            uri = URI.parse(public_link)
+            args = URI.decode_www_form(uri.query).each_with_object({}){|v, h|h[v.first] = v.last; }
+            Log.dump(:args, args)
+            context = args['context']
+            raise 'missing context' if context.nil?
+            @pub_link_context = JSON.parse(Base64.decode64(context))
+            Log.dump(:@pub_link_context, @pub_link_context)
+            @api_v5 = Rest.new({
+              base_url: faspex5_api_v5_url,
+              headers:  {'Passcode' => @pub_link_context['passcode']}
+            })
           when :boot
             # the password here is the token copied directly from browser in developer mode
             @api_v5 = Rest.new({
@@ -78,17 +106,18 @@ module Aspera
                 jwt:          {
                   payload:         {
                     iss: app_client_id,    # issuer
-                    aud: app_client_id,    # audience TODO: ???
-                    sub: "user:#{options.get_option(:username, is_type: :mandatory)}" # subject also "client:#{app_client_id}" + auth user/pass
+                    aud: app_client_id,    # audience (this field is not clear...)
+                    sub: "user:#{options.get_option(:username, is_type: :mandatory)}" # subject is a user
                   },
-                  # auth:                {type: :basic, options.get_option(:username,is_type: :mandatory), options.get_option(:password,is_type: :mandatory),
                   private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, is_type: :mandatory), options.get_option(:passphrase)),
                   headers:         {typ: 'JWT'}
                 }
               }})
+          else raise 'Unexpected case for option: auth'
           end
         end
 
+        # if recipient is just an email, then convert to expected API hash : name and type
         def normalize_recipients(parameters)
           return unless parameters.key?('recipients')
           raise 'Field recipients must be an Array' unless parameters['recipients'].is_a?(Array)
@@ -96,7 +125,7 @@ module Aspera
             # if just a string, assume it is the name
             if recipient_data.is_a?(String)
               result = @api_v5.read('contacts', {q: recipient_data, context: 'packages', type: [Rest::ARRAY_PARAMS, *RECIPIENT_TYPES]})[:data]
-              raise "No matching contact for #{recipient_data}" if result.empty?
+              raise "No matching contact for #{recipient_data}" if 0.eql?(result['total_count'])
               raise "Multiple matching contact for #{recipient_data} : #{result['contacts'].map{|i|i['name']}.join(', ')}" unless 1.eql?(result['total_count'])
               matched = result['contacts'].first
               recipient_data = {
@@ -109,7 +138,8 @@ module Aspera
           end
         end
 
-        def wait_for_complete_upload(id)
+        # wait for package status to be in provided list
+        def wait_package_status(id, status_list=PACKAGE_TERMINATED)
           parameters = options.get_option(:value)
           spinner = nil
           progress = nil
@@ -133,22 +163,70 @@ module Aspera
             else
               progress.progress = status['bytes_written'].to_i
             end
-            break if PACKAGE_TERMINATED.include?(status['upload_status'])
+            break if status_list.include?(status['upload_status'])
             sleep(0.5)
           end
           status['id'] = id
           return status
         end
 
-        def lookup_entity(entity_type, property, value)
-          # TODO: what if too many, use paging ?
-          all = @api_v5.read(entity_type)[:data][entity_type]
-          found = all.find{|i|i[property].eql?(value)}
-          raise "No #{entity_type} with #{property} = #{value}" if found.nil?
-          return found
+        # get a list of all entities of a given type
+        # @param entity_type [String] the type of entity to list
+        # @param query [Hash] additional query parameters
+        # @param prefix [String] optional prefix to add to the path (nil or empty string: no prefix)
+        def list_entities(entity_type, query: {}, prefix: nil)
+          path = entity_type
+          path = "#{prefix}/#{path}" unless prefix.nil? || prefix.empty?
+          result = []
+          offset = 0
+          max_items = query.delete(MAX_ITEMS)
+          remain_pages = query.delete(MAX_PAGES)
+          # merge default parameters, by default 100 per page
+          query = {'limit'=> 100}.merge(query)
+          loop do
+            query['offset'] = offset
+            page = @api_v5.read(path, query)[:data]
+            result.concat(page[entity_type])
+            # reach the limit set by user ?
+            if !max_items.nil? && (result.length >= max_items)
+              result = result.slice(0, max_items)
+              break
+            end
+            break if result.length >= page['total_count']
+            remain_pages -= 1 unless remain_pages.nil?
+            break if remain_pages == 0
+            offset += page[entity_type].length
+          end
+          return result
+        end
+
+        # lookup an entity id from its name
+        def lookup_name_to_id(entity_type, name)
+          found = list_entities(entity_type, query: {'q'=> name}).select{|i|i['name'].eql?(name)}
+          case found.length
+          when 0 then raise "No #{entity_type} with name = #{name}"
+          when 1 then return found.first['id']
+          else raise "Multiple #{entity_type} with name = #{name}"
+          end
         end
 
-        ACTIONS = %i[health version user bearer_token package shared_folders admin gateway postprocessing].freeze
+        # translate box name to API prefix (with ending slash)
+        def box_to_prefix(box)
+          return \
+          case box
+          when VAL_ALL then ''
+          when *API_MAILBOXES then box
+          else "shared_inboxes/#{lookup_name_to_id('shared_inboxes', box)}"
+          end
+        end
+
+        # list all packages with optional filter
+        def list_packages
+          parameters = options.get_option(:value) || {}
+          return list_entities('packages', query: parameters, prefix: box_to_prefix(options.get_option(:box)))
+        end
+
+        ACTIONS = %i[health version user bearer_token packages shared_folders admin gateway postprocessing].freeze
 
         def execute_action
           command = options.get_next_command(ACTIONS)
@@ -180,22 +258,47 @@ module Aspera
             end
           when :bearer_token
             return {type: :text, data: @api_v5.oauth_token}
-          when :package
-            command = options.get_next_command(%i[list show send receive status])
+          when :packages
+            command = options.get_next_command(%i[list show browse status delete send receive])
             case command
             when :list
-              parameters = options.get_option(:value)
               return {
                 type:   :object_list,
-                data:   @api_v5.read('packages', parameters)[:data]['packages'],
+                data:   list_packages,
                 fields: %w[id title release_date total_bytes total_files created_time state]
               }
             when :show
-              id = instance_identifier
+              id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
+              id ||= instance_identifier
               return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
+            when :browse
+              id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
+              id ||= instance_identifier
+              path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
+              # TODO: support multi-page listing ?
+              params = {
+                # recipient_user_id: 25,
+                # offset:            0,
+                # limit:             25
+              }
+              result = @api_v5.call({
+                operation:   'POST',
+                subpath:     "packages/#{id}/files/received",
+                headers:     {'Accept' => 'application/json'},
+                url_params:  params,
+                json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
+              formatter.display_item_count(result['item_count'], result['total_count'])
+              return {type: :object_list, data: result['items']}
             when :status
-              status = wait_for_complete_upload(instance_identifier)
+              status = wait_package_status(instance_identifier)
               return {type: :single_object, data: status}
+            when :delete
+              ids = instance_identifier
+              ids = [ids] unless ids.is_a?(Array)
+              raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
+              # API returns 204, empty on success
+              @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
+              return Main.result_status('Package(s) deleted')
             when :send
               parameters = options.get_option(:value, is_type: :mandatory)
               raise CliBadArgument, 'Value must be Hash, refer to API' unless parameters.is_a?(Hash)
@@ -211,11 +314,12 @@ module Aspera
                   url_params:  {transfer_type: TRANSFER_CONNECT},
                   json_params: {paths: transfer.source_list}
                 )[:data]
+                # well, we asked a TS for connect, but we actually want a generic one
                 transfer_spec.delete('authentication')
                 return Main.result_transfer(transfer.start(transfer_spec))
               else
                 if !shared_folder.to_i.to_s.eql?(shared_folder)
-                  shared_folder = lookup_entity('shared_folders', 'name', shared_folder)['id']
+                  shared_folder = lookup_name_to_id('shared_folders', shared_folder)
                 end
                 transfer_request = {shared_folder_id: shared_folder, paths: transfer.source_list}
                 # start remote transfer and get first status
@@ -223,40 +327,50 @@ module Aspera
                 result['id'] = package['id']
                 unless result['status'].eql?('completed')
                   formatter.display_status("Package #{package['id']}")
-                  result = wait_for_complete_upload(package['id'])
+                  result = wait_package_status(package['id'])
                 end
                 return {type: :single_object, data: result}
               end
             when :receive
-              pkg_type = 'received'
-              pack_id = instance_identifier
-              package_ids = [pack_id]
-              skip_ids_data = []
+              # prepare persistency if needed
               skip_ids_persistency = nil
               if options.get_option(:once_only, is_type: :mandatory)
                 # read ids from persistency
                 skip_ids_persistency = PersistencyActionOnce.new(
                   manager: @agents[:persistency],
-                  data:    skip_ids_data,
+                  data:    [],
                   id:      IdGenerator.from_list([
                     'faspex_recv',
                     options.get_option(:url, is_type: :mandatory),
                     options.get_option(:username, is_type: :mandatory),
-                    pkg_type]))
+                    PACKAGE_TYPE_RECEIVED]))
               end
-              if VAL_ALL.eql?(pack_id)
-                # TODO: if packages have same name, they will overwrite
-                parameters = options.get_option(:value)
-                parameters ||= {'type' => 'received', 'subtype' => 'mypackages', 'limit' => 1000}
-                raise CliBadArgument, 'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
-                package_ids = @api_v5.read('packages', parameters)[:data]['packages'].map{|p|p['id']}
-                package_ids.reject!{|i|skip_ids_data.include?(i)}
+              # one or several packages
+              package_ids = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
+              package_ids ||= instance_identifier
+              case package_ids
+              when PACKAGE_ALL_INIT
+                raise 'Only with option once_only' unless skip_ids_persistency
+                skip_ids_persistency.data.clear.concat(list_packages.map{|p|p['id']})
+                skip_ids_persistency.save
+                return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
+              when VAL_ALL
+                # TODO: if packages have same name, they will overwrite ?
+                package_ids = list_packages.map{|p|p['id']}
+                Log.dump(:package_ids, package_ids)
+                Log.dump(:package_ids, skip_ids_persistency.data)
+                package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
+                Log.dump(:package_ids, package_ids)
               end
+              # a single id was provided
+              # TODO: check package_ids is a list of strings
+              package_ids = [package_ids] if package_ids.is_a?(String)
               result_transfer = []
               package_ids.each do |pkg_id|
+                formatter.display_status("Receiving package #{pkg_id}")
                 param_file_list = {}
                 begin
-                  param_file_list['paths'] = transfer.source_list
+                  param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
                 rescue Aspera::Cli::CliBadArgument
                   # paths is optional
                 end
@@ -265,16 +379,19 @@ module Aspera
                   operation:   'POST',
                   subpath:     "packages/#{pkg_id}/transfer_spec/download",
                   headers:     {'Accept' => 'application/json'},
-                  url_params:  {transfer_type: TRANSFER_CONNECT, type: pkg_type},
+                  url_params:  {transfer_type: TRANSFER_CONNECT, type: PACKAGE_TYPE_RECEIVED},
                   json_params: param_file_list
                 )[:data]
+                # delete flag for Connect Client
                 transfer_spec.delete('authentication')
                 statuses = transfer.start(transfer_spec)
                 result_transfer.push({'package' => pkg_id, Main::STATUS_FIELD => statuses})
                 # skip only if all sessions completed
-                skip_ids_data.push(pkg_id) if TransferAgent.session_status(statuses).eql?(:success)
+                if TransferAgent.session_status(statuses).eql?(:success) && skip_ids_persistency
+                  skip_ids_persistency.data.push(pkg_id)
+                  skip_ids_persistency.save
+                end
               end
-              skip_ids_persistency&.save
               return Main.result_transfer_multiple(result_transfer)
             end # case package
           when :shared_folders
@@ -301,7 +418,7 @@ module Aspera
               end
             end
           when :admin
-            case options.get_next_command(%i[resource])
+            case options.get_next_command(%i[resource smtp])
             when :resource
               res_type = options.get_next_command(%i[accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs metadata_profiles
                                                      email_notifications])
@@ -325,6 +442,22 @@ module Aspera
                 adm_api = Rest.new(@api_v5.params.merge({base_url: @faspex5_api_auth_url}))
               end
               return entity_action(adm_api, res_path, item_list_key: list_key, display_fields: display_fields, id_as_arg: id_as_arg)
+            when :smtp
+              smtp_path = 'configuration/smtp'
+              case options.get_next_command(%i[show create modify delete test])
+              when :show
+                return { type: :single_object, data: @api_v5.read(smtp_path)[:data] }
+              when :create
+                return { type: :single_object, data: @api_v5.create(smtp_path, options.get_option(:value, is_type: :mandatory))[:data] }
+              when :modify
+                return { type: :single_object, data: @api_v5.modify(smtp_path, options.get_option(:value, is_type: :mandatory))[:data] }
+              when :delete
+                return { type: :single_object, data: @api_v5.delete(smtp_path)[:data] }
+              when :test
+                test_data = options.get_next_argument('Email or test data, see API')
+                test_data = {test_email_recipient: test_data} if test_data.is_a?(String)
+                return { type: :single_object, data: @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data] }
+              end
             end
           when :gateway
             require 'aspera/faspex_gw'