aspera-cli 4.14.0 → 4.15.0

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# spellchecker: ignore workgroups,mypackages
+# spellchecker: ignore workgroups mypackages passcode
 
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
@@ -8,7 +8,6 @@ require 'aspera/id_generator'
 require 'aspera/nagios'
 require 'aspera/environment'
 require 'securerandom'
-require 'ruby-progressbar'
 require 'tty-spinner'
 
 module Aspera
@@ -22,78 +21,130 @@ module Aspera
         API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
         PACKAGE_ALL_INIT = 'INIT'
         PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
-        ADMIN_RESOURCES = %i[accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs metadata_profiles
-                             email_notifications].freeze
-        private_constant(*%i[RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE])
+        # Faspex API v5: get transfer spec for connect
+        TRANSFER_CONNECT = 'connect'
+        ADMIN_RESOURCES = %i[
+          accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
+          metadata_profiles email_notifications
+        ].freeze
+        JOB_RUNNING = %w[queued working].freeze
+        STANDARD_PATH = '/aspera/faspex'
+        private_constant(*%i[JOB_RUNNING RECIPIENT_TYPES PACKAGE_TERMINATED API_DETECT API_LIST_MAILBOX_TYPES PACKAGE_SEND_FROM_REMOTE_SOURCE])
         class << self
-          def detect(base_url)
-            api = Rest.new(base_url: base_url, redirect_max: 1)
-            result = api.read(API_DETECT)
-            if result[:http].code.start_with?('2') && result[:http].body.strip.empty?
-              suffix_length = -2 - API_DETECT.length
+          def application_name
+            'Faspex'
+          end
+
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+
+            urls.each do |base_url|
+              next unless base_url.start_with?('https://')
+              api = Rest.new(base_url: base_url, redirect_max: 1)
+              result = api.read(API_DETECT)
+              next unless result[:http].code.start_with?('2') && result[:http].body.strip.empty?
+              url_length = -2 - API_DETECT.length
+              # take redirect if any
               return {
                 version: result[:http]['x-ibm-aspera'] || '5',
-                url:     result[:http].uri.to_s[0..suffix_length],
-                name:    'Faspex 5'
+                url:     result[:http].uri.to_s[0..url_length]
               }
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
             end
             return nil
           end
-        end
 
-        # Faspex API v5: get transfer spec for connect
-        TRANSFER_CONNECT = 'connect'
+          def wizard(object:, private_key_path:, pub_key_pem:)
+            options = object.options
+            formatter = object.formatter
+            instance_url = options.get_option(:url, mandatory: true)
+            wiz_username = options.get_option(:username, mandatory: true)
+            raise "Username shall be an email in Faspex: #{wiz_username}" if !(wiz_username =~ /\A[\w+\-.]+@[a-z\d\-.]+\.[a-z]+\z/i)
+            if options.get_option(:client_id).nil? || options.get_option(:client_secret).nil?
+              formatter.display_status('Ask the ascli client id and secret to your Administrator.'.red)
+              formatter.display_status("Admin should login to: #{instance_url}")
+              OpenApplication.instance.uri(instance_url)
+              formatter.display_status('Navigate to: 𓃑  → Admin → Configurations → API clients')
+              formatter.display_status('Create an API client with:')
+              formatter.display_status('- name: ascli')
+              formatter.display_status('- JWT: enabled')
+              formatter.display_status("Then, logged in as #{wiz_username.red} go to your profile:")
+              formatter.display_status('👤 → Account Settings → Preferences -> Public Key in PEM:')
+              formatter.display_status(pub_key_pem)
+              formatter.display_status('Once set, fill in the parameters:')
+            end
+            return {
+              preset_value: {
+                url:           instance_url,
+                username:      wiz_username,
+                auth:          :jwt.to_s,
+                private_key:   "@file:#{private_key_path}",
+                client_id:     options.get_option(:client_id, mandatory: true),
+                client_secret: options.get_option(:client_secret, mandatory: true)
+              },
+              test_args:    'user profile show'
+            }
+          end
+
+          def public_link?(url)
+            url.include?('/public/')
+          end
+        end
 
         def initialize(env)
           super(env)
           options.declare(:client_id, 'OAuth client identifier')
           options.declare(:client_secret, 'OAuth client secret')
           options.declare(:redirect_uri, 'OAuth redirect URI for web authentication')
-          options.declare(:auth, 'OAuth type of authentication', values: %i[boot link].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
+          options.declare(:auth, 'OAuth type of authentication', values: %i[boot].concat(Oauth::STD_AUTH_TYPES), default: :jwt)
           options.declare(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
           options.declare(:passphrase, 'OAuth JWT RSA private key passphrase')
-          options.declare(:link, 'Public link authorization (specific operations)')
-          options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{VAL_ALL}", default: 'inbox')
+          options.declare(:box, "Package inbox, either shared inbox name or one of #{API_LIST_MAILBOX_TYPES} or #{ExtendedValue::ALL}", default: 'inbox')
           options.declare(:shared_folder, 'Send package with files from shared folder')
-          options.declare(:group_type, 'Shared inbox or workgroup', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
+          options.declare(:group_type, 'Type of shared box', values: %i[shared_inboxes workgroups], default: :shared_inboxes)
           options.parse_options!
         end
 
+        def api_url
+          return "#{@faspex5_api_base_url}/api/v5"
+        end
+
+        def auth_api_url
+          return "#{@faspex5_api_base_url}/auth"
+        end
+
         def set_api
-          public_link = options.get_option(:link)
-          unless public_link.nil?
-            @faspex5_api_base_url = public_link.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
-            options.set_option(:auth, :link)
-          end
-          @faspex5_api_base_url ||= options.get_option(:url, mandatory: true).gsub(%r{/+$}, '')
-          @faspex5_api_auth_url = "#{@faspex5_api_base_url}/auth"
-          faspex5_api_v5_url = "#{@faspex5_api_base_url}/api/v5"
-          case options.get_option(:auth, mandatory: true)
-          when :link
-            uri = URI.parse(public_link)
-            args = URI.decode_www_form(uri.query).each_with_object({}){|v, h|h[v.first] = v.last; }
-            Log.dump(:args, args)
-            context = args['context']
-            raise 'missing context' if context.nil?
-            @pub_link_context = JSON.parse(Base64.decode64(context))
-            Log.dump(:@pub_link_context, @pub_link_context)
+          # get endpoint, remove unnecessary trailing slashes
+          @faspex5_api_base_url = options.get_option(:url, mandatory: true).gsub(%r{/+$}, '')
+          auth_type = self.class.public_link?(@faspex5_api_base_url) ? :public_link : options.get_option(:auth, mandatory: true)
+          case auth_type
+          when :public_link
+            encoded_context = Rest.decode_query(URI.parse(@faspex5_api_base_url).query)['context']
+            raise 'Bad faspex5 public link, missing context in query' if encoded_context.nil?
+            @pub_link_context = JSON.parse(Base64.decode64(encoded_context))
+            Log.log.trace1{Log.dump(:@pub_link_context, @pub_link_context)}
+            # ok, we have the additional parameters, get the base url
+            @faspex5_api_base_url = @faspex5_api_base_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               headers:  {'Passcode' => @pub_link_context['passcode']}
             })
           when :boot
             # the password here is the token copied directly from browser in developer mode
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               headers:  {'Authorization' => options.get_option(:password, mandatory: true)}
             })
           when :web
             # opens a browser and ask user to auth using web
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               auth:     {
                 type:         :oauth2,
-                base_url:     @faspex5_api_auth_url,
+                base_url:     auth_api_url,
                 grant_method: :web,
                 client_id:    options.get_option(:client_id, mandatory: true),
                 web:          {redirect_uri: options.get_option(:redirect_uri, mandatory: true)}
@@ -101,10 +152,10 @@ module Aspera
           when :jwt
             app_client_id = options.get_option(:client_id, mandatory: true)
             @api_v5 = Rest.new({
-              base_url: faspex5_api_v5_url,
+              base_url: api_url,
               auth:     {
                 type:         :oauth2,
-                base_url:     @faspex5_api_auth_url,
+                base_url:     auth_api_url,
                 grant_method: :jwt,
                 client_id:    app_client_id,
                 jwt:          {
@@ -147,32 +198,45 @@ module Aspera
 
         # wait for package status to be in provided list
         def wait_package_status(id, status_list: PACKAGE_TERMINATED)
-          spinner = nil
-          progress = nil
-          while true
+          total_sent = false
+          loop do
             status = @api_v5.read("packages/#{id}/upload_details")[:data]
+            status['id'] = id
             # user asked to not follow
-            break if status_list.nil? || status_list.include?(status['upload_status'])
+            return status if status_list.nil?
             if status['upload_status'].eql?('submitted')
-              if spinner.nil?
-                spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
-                spinner.start
-              end
-              spinner.update(title: status['upload_status'])
-              spinner.spin
-            elsif progress.nil?
-              progress = ProgressBar.create(
-                format:     '%a %B %p%% %r Mbps %e',
-                rate_scale: lambda{|rate|rate / Environment::BYTES_PER_MEBIBIT},
-                title:      'progress',
-                total:      status['bytes_total'].to_i)
+              config.progress_bar&.event(session_id: nil, type: :pre_start, info: status['upload_status'])
+            elsif !total_sent
+              config.progress_bar&.event(session_id: id, type: :session_start)
+              config.progress_bar&.event(session_id: id, type: :session_size, info: status['bytes_total'].to_i)
+              total_sent = true
             else
-              progress.progress = status['bytes_written'].to_i
+              config.progress_bar&.event(session_id: id, type: :transfer, info: status['bytes_written'].to_i)
+            end
+            if status_list.include?(status['upload_status'])
+              # if status['upload_status'].eql?('completed')
+              config.progress_bar&.event(session_id: id, type: :end)
+              return status
+              # end
+            end
+            sleep(1.0)
+          end
+        end
+
+        def wait_for_job(job_id)
+          spinner = nil
+          loop do
+            status = @api_v5.read("jobs/#{job_id}", {type: :formatted})[:data]
+            return status unless JOB_RUNNING.include?(status['status'])
+            if spinner.nil?
+              spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
+              spinner.start
             end
+            spinner.update(title: status['status'])
+            spinner.spin
             sleep(0.5)
           end
-          status['id'] = id
-          return status
+          raise 'internal error'
         end
 
         # get a (full or partial) list of all entities of a given type
@@ -228,7 +292,7 @@ module Aspera
           box = options.get_option(:box)
           api_path =
             case box
-            when VAL_ALL then '' # only admin can list all packages globally
+            when ExtendedValue::ALL then '' # only admin can list all packages globally
             when *API_LIST_MAILBOX_TYPES then box
             else
               group_type = options.get_option(:group_type)
@@ -240,7 +304,7 @@ module Aspera
             path: api_path).select(&filter)
         end
 
-        def package_receive
+        def package_receive(package_ids)
           # prepare persistency if needed
           skip_ids_persistency = nil
           if options.get_option(:once_only, mandatory: true)
@@ -253,50 +317,43 @@ module Aspera
                 options.get_option(:url, mandatory: true),
                 options.get_option(:username, mandatory: true)]))
           end
-          package_ids =
-            if @pub_link_context&.key?('package_id')
-              @pub_link_context['package_id']
-            else
-              # one or several packages
-              instance_identifier
-            end
           case package_ids
           when PACKAGE_ALL_INIT
             raise 'Only with option once_only' unless skip_ids_persistency
             skip_ids_persistency.data.clear.concat(list_packages_with_filter.map{|p|p['id']})
             skip_ids_persistency.save
             return Main.result_status("Initialized skip for #{skip_ids_persistency.data.count} package(s)")
-          when VAL_ALL
+          when ExtendedValue::ALL
             # TODO: if packages have same name, they will overwrite ?
             package_ids = list_packages_with_filter.map{|p|p['id']}
-            Log.dump(:package_ids, package_ids)
-            Log.dump(:package_ids, skip_ids_persistency.data)
+            Log.log.debug{Log.dump(:package_ids, package_ids)}
+            Log.log.debug{Log.dump(:skip_ids, skip_ids_persistency.data)}
             package_ids.reject!{|i|skip_ids_persistency.data.include?(i)} if skip_ids_persistency
-            Log.dump(:package_ids, package_ids)
+            Log.log.debug{Log.dump(:package_ids, package_ids)}
           end
           # a single id was provided
           # TODO: check package_ids is a list of strings
           package_ids = [package_ids] if package_ids.is_a?(String)
           result_transfer = []
+          param_file_list = {}
+          begin
+            param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
+          rescue Cli::BadArgument
+            # paths is optional
+          end
+          download_params = {
+            type:          'received',
+            transfer_type: TRANSFER_CONNECT
+          }
+          box = options.get_option(:box)
+          case box
+          when /outbox/ then download_params[:type] = 'sent'
+          when *API_LIST_MAILBOX_TYPES then nil # nothing to do
+          else # shared inbox / workgroup
+            download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
+          end
           package_ids.each do |pkg_id|
             formatter.display_status("Receiving package #{pkg_id}")
-            param_file_list = {}
-            begin
-              param_file_list['paths'] = transfer.source_list.map{|source|{'path'=>source}}
-            rescue Aspera::Cli::CliBadArgument
-              # paths is optional
-            end
-            download_params = {
-              type:          'received',
-              transfer_type: TRANSFER_CONNECT
-            }
-            box = options.get_option(:box)
-            case box
-            when /outbox/ then download_params[:type] = 'sent'
-            when *API_LIST_MAILBOX_TYPES then nil # nothing to do
-            else # shared inbox / workgroup
-              download_params[:recipient_workgroup_id] = lookup_entity_by_field(type: options.get_option(:group_type), value: box)['id']
-            end
             # TODO: allow from sent as well ?
             transfer_spec = @api_v5.call(
               operation:   'POST',
@@ -319,21 +376,15 @@ module Aspera
         end
 
         def package_action
-          command = options.get_next_command(%i[list show browse status delete send receive])
+          command = options.get_next_command(%i[show browse status delete receive send list])
+          package_id =
+            if %i[receive show browse status delete].include?(command)
+              @pub_link_context&.key?('package_id') ? @pub_link_context['package_id'] : instance_identifier
+            end
           case command
-          when :list
-            return {
-              type:   :object_list,
-              data:   list_packages_with_filter,
-              fields: %w[id title release_date total_bytes total_files created_time state]
-            }
           when :show
-            id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-            id ||= instance_identifier
-            return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
+            return {type: :single_object, data: @api_v5.read("packages/#{package_id}")[:data]}
           when :browse
-            id = @pub_link_context['package_id'] if @pub_link_context&.key?('package_id')
-            id ||= instance_identifier
             path = options.get_next_argument('path', expected: :single, mandatory: false) || '/'
             # TODO: support multi-page listing ?
             params = {
@@ -343,29 +394,31 @@ module Aspera
             }
             result = @api_v5.call({
               operation:   'POST',
-              subpath:     "packages/#{id}/files/received",
+              subpath:     "packages/#{package_id}/files/received",
               headers:     {'Accept' => 'application/json'},
               url_params:  params,
               json_params: {'path' => path, 'filters' => {'basenames'=>[]}}})[:data]
             formatter.display_item_count(result['item_count'], result['total_count'])
             return {type: :object_list, data: result['items']}
           when :status
-            status = wait_package_status(instance_identifier, status_list: nil)
+            status = wait_package_status(package_id, status_list: nil)
             return {type: :single_object, data: status}
           when :delete
-            ids = instance_identifier
+            ids = package_id
             ids = [ids] unless ids.is_a?(Array)
             raise 'Package identifier must be a single id or an Array' unless ids.is_a?(Array) && ids.all?(String)
             # API returns 204, empty on success
             @api_v5.call({operation: 'DELETE', subpath: 'packages', headers: {'Accept' => 'application/json'}, json_params: {ids: ids}})
             return Main.result_status('Package(s) deleted')
+          when :receive
+            return package_receive(package_id)
           when :send
-            parameters = value_create_modify(command: command, type: Hash)
+            parameters = value_create_modify(command: command)
             normalize_recipients(parameters)
             package = @api_v5.create('packages', parameters)[:data]
             shared_folder = options.get_option(:shared_folder)
             if shared_folder.nil?
-              # TODO: option to send from remote source or httpgw
+              # send from local files
               transfer_spec = @api_v5.call(
                 operation:   'POST',
                 subpath:     "packages/#{package['id']}/transfer_spec/upload",
@@ -377,6 +430,7 @@ module Aspera
               transfer_spec.delete('authentication')
               return Main.result_transfer(transfer.start(transfer_spec))
             else
+              # send from remote shared folder
               if (m = shared_folder.match(REGEX_LOOKUP_ID_BY_FIELD))
                 shared_folder = lookup_entity_by_field(type: 'shared_folders', value: m[2])['id']
               end
@@ -390,8 +444,12 @@ module Aspera
               end
               return {type: :single_object, data: result}
             end
-          when :receive
-            return package_receive
+          when :list
+            return {
+              type:   :object_list,
+              data:   list_packages_with_filter,
+              fields: %w[id title release_date total_bytes total_files created_time state]
+            }
           end # case package
         end
 
@@ -474,10 +532,10 @@ module Aspera
                 list_key = false
                 id_as_arg = 'type'
               when :accounts
-                display_fields = [:all_but, 'user_profile_data_attributes']
+                display_fields = Formatter.all_but('user_profile_data_attributes')
               when :oauth_clients
-                display_fields = [:all_but, 'public_key']
-                adm_api = Rest.new(@api_v5.params.merge({base_url: @faspex5_api_auth_url}))
+                display_fields = Formatter.all_but('public_key')
+                adm_api = Rest.new(@api_v5.params.merge({base_url: auth_api_url}))
               when :shared_inboxes, :workgroups
                 available_commands.push(:members, :saml_groups, :invite_external_collaborator)
               end
@@ -533,20 +591,23 @@ module Aspera
               when :show
                 return { type: :single_object, data: @api_v5.read(smtp_path)[:data] }
               when :create
-                return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd, type: Hash))[:data] }
+                return { type: :single_object, data: @api_v5.create(smtp_path, value_create_modify(command: smtp_cmd))[:data] }
               when :modify
-                return { type: :single_object, data: @api_v5.modify(smtp_path, value_create_modify(command: smtp_cmd, type: Hash))[:data] }
+                return { type: :single_object, data: @api_v5.update(smtp_path, value_create_modify(command: smtp_cmd))[:data] }
               when :delete
                 return { type: :single_object, data: @api_v5.delete(smtp_path)[:data] }
               when :test
                 test_data = options.get_next_argument('Email or test data, see API')
                 test_data = {test_email_recipient: test_data} if test_data.is_a?(String)
-                return { type: :single_object, data: @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data] }
+                creation = @api_v5.create(File.join(smtp_path, 'test'), test_data)[:data]
+                result = wait_for_job(creation['job_id'])
+                result['serialized_args'] = JSON.parse(result['serialized_args']) rescue result['serialized_args']
+                return { type: :single_object, data: result }
               end
             end
           when :gateway
             require 'aspera/faspex_gw'
-            url = value_create_modify(type: String)
+            url = value_create_modify(command: command, type: String)
             uri = URI.parse(url)
             server = WebServerSimple.new(uri)
             server.mount(uri.path, Faspex4GWServlet, @api_v5, nil)
@@ -559,7 +620,7 @@ module Aspera
             return Main.result_status('Gateway terminated')
           when :postprocessing
             require 'aspera/faspex_postproc' # cspell:disable-line
-            parameters = value_create_modify(type: Hash)
+            parameters = value_create_modify(command: command)
             parameters = parameters.symbolize_keys
             raise 'Missing key: url' unless parameters.key?(:url)
             uri = URI.parse(parameters[:url])
@@ -576,36 +637,6 @@ module Aspera
             return Main.result_status('Gateway terminated')
           end # case command
         end # action
-
-        def wizard(params)
-          if params[:prepare]
-            # if not defined by user, generate unique name
-            params[:preset_name] ||= [params[:plugin_sym]].concat(URI.parse(params[:instance_url]).host.gsub(/[^a-z0-9.]/, '').split('.')).join('_')
-            params[:need_private_key] = true
-            return
-          end
-          formatter.display_status('Ask the ascli client id and secret to your Administrator, or ask them to go to:'.red)
-          OpenApplication.instance.uri(params[:instance_url])
-          formatter.display_status('Then: 𓃑  → Admin → Configurations → API clients')
-          formatter.display_status('Create an API client with:')
-          formatter.display_status('- name: ascli')
-          formatter.display_status('- JWT: enabled')
-          formatter.display_status('Then, logged in as user go to your profile:')
-          formatter.display_status('👤 → Account Settings → Preferences -> Public Key in PEM:')
-          formatter.display_status(params[:pub_key_pem])
-          formatter.display_status('Once set, fill in the parameters:')
-          return {
-            preset_value: {
-              url:           params[:instance_url],
-              username:      options.get_option(:username, mandatory: true),
-              auth:          :jwt.to_s,
-              private_key:   '@file:' + params[:private_key_path],
-              client_id:     options.get_option(:client_id, mandatory: true),
-              client_secret: options.get_option(:client_secret, mandatory: true)
-            },
-            test_args:    "#{params[:plugin_sym]} user profile show"
-          }
-        end
       end # Faspex5
     end # Plugins
   end # Cli