aspera-cli 4.15.0 → 4.17.0

data/lib/aspera/cos_node.rb

@@ -1,93 +0,0 @@
-# frozen_string_literal: true
-
-require 'aspera/log'
-require 'aspera/rest'
-require 'aspera/oauth'
-require 'xmlsimple'
-
-module Aspera
-  class CosNode < Aspera::Node
-    class << self
-      def parameters_from_svc_credentials(service_credentials, bucket_region)
-        # check necessary contents
-        raise 'service_credentials must be a Hash' unless service_credentials.is_a?(Hash)
-        %w[apikey resource_instance_id endpoints].each do |field|
-          raise "service_credentials must have a field: #{field}" unless service_credentials.key?(field)
-        end
-        Aspera::Log.dump('service_credentials', service_credentials)
-        # read endpoints from service provided in service credentials
-        endpoints = Aspera::Rest.new({base_url: service_credentials['endpoints']}).read('')[:data]
-        Aspera::Log.dump('endpoints', endpoints)
-        storage_endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
-        raise "no such region: #{bucket_region}" if storage_endpoint.nil?
-        return {
-          instance_id:      service_credentials['resource_instance_id'],
-          service_api_key:  service_credentials['apikey'],
-          storage_endpoint: "https://#{storage_endpoint}"
-        }
-      end
-    end
-    IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
-    TOKEN_FIELD = 'delegated_refresh_token'
-
-    def initialize(bucket_name, storage_endpoint, instance_id, api_key, auth_url= IBM_CLOUD_TOKEN_URL)
-      @auth_url = auth_url
-      @api_key = api_key
-      s3_api = Aspera::Rest.new({
-        base_url:       storage_endpoint,
-        not_auth_codes: %w[401 403], # error codes when not authorized
-        headers:        {'ibm-service-instance-id' => instance_id},
-        auth:           {
-          type:         :oauth2,
-          base_url:     @auth_url,
-          grant_method: :generic,
-          generic:      {
-            grant_type:    'urn:ibm:params:oauth:grant-type:apikey',
-            response_type: 'cloud_iam',
-            apikey:        @api_key
-          }}})
-      # read FASP connection information for bucket
-      xml_result_text = s3_api.call(
-        operation: 'GET',
-        subpath: bucket_name,
-        headers: {'Accept' => 'application/xml'},
-        url_params: {'faspConnectionInfo' => nil}
-      )[:http].body
-      ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
-      Aspera::Log.dump('ats_info', ats_info)
-      @storage_credentials = {
-        'type'  => 'token',
-        'token' => {TOKEN_FIELD => nil}
-      }
-      super(
-        params: {
-          base_url: ats_info['ATSEndpoint'],
-          auth:     {
-            type:     :basic,
-            username: ats_info['AccessKey']['Id'],
-            password: ats_info['AccessKey']['Secret']}},
-        add_tspec: {'tags'=>{Fasp::TransferSpec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}})
-      # update storage_credentials AND Rest params
-      generate_token
-    end
-
-    # potentially call this if delegated token is expired
-    def generate_token
-      # OAuth API to get delegated token
-      delegated_oauth = Oauth.new({
-        type:         :oauth2,
-        base_url:     @auth_url,
-        token_field:  TOKEN_FIELD,
-        grant_method: :generic,
-        generic:      {
-          grant_type:          'urn:ibm:params:oauth:grant-type:apikey',
-          response_type:       'delegated_refresh_token',
-          apikey:              @api_key,
-          receiver_client_ids: 'aspera_ats'
-        }})
-      # get delegated token to be placed in rest call header and in transfer tags
-      @storage_credentials['token'][TOKEN_FIELD] = Oauth.bearer_extract(delegated_oauth.get_authorization)
-      @params[:headers] = {'X-Aspera-Storage-Credentials' => JSON.generate(@storage_credentials)}
-    end
-  end
-end

data/lib/aspera/fasp/agent_aspera.rb

@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-require 'aspera/fasp/agent_base'
-require 'aspera/rest'
-require 'aspera/json_rpc'
-require 'aspera/open_application'
-require 'securerandom'
-
-module Aspera
-  module Fasp
-    class AgentAspera < Aspera::Fasp::AgentBase
-      # try twice the main init url in sequence
-      START_URIS = ['aspera://']
-      # delay between each try to start connect
-      SLEEP_SEC_BETWEEN_RETRY = 3
-      private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
-      def initialize(options)
-        @application_id = SecureRandom.uuid
-        super(options)
-        raise 'Using client requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
-        method_index = 0
-        begin
-          @client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
-          client_info = @client_app_api.get_info
-          Log.log.debug{Log.dump(:client_version, client_info)}
-          # my_transfer_id = '0513fe85-65cf-465b-ad5f-18fd40d8c69f'
-          # @client_app_api.get_all_transfers({app_id: @application_id})
-          # @client_app_api.get_transfer(app_id: @application_id, transfer_id: my_transfer_id)
-          # @client_app_api.start_transfer(app_id: @application_id,transfer_spec: {})
-          # @client_app_api.remove_transfer
-          # @client_app_api.stop_transfer
-          # @client_app_api.modify_transfer
-          # @client_app_api.show_directory({app_id: @application_id, transfer_id: my_transfer_id})
-          # @client_app_api.get_files_list({app_id: @application_id, transfer_id: my_transfer_id})
-          Log.log.info('Client was reached') if method_index > 0
-        rescue StandardError => e # Errno::ECONNREFUSED
-          start_url = START_URIS[method_index]
-          method_index += 1
-          raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
-          Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
-          if !OpenApplication.uri_graphical(start_url)
-            OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
-            raise StandardError, 'Connect is not installed'
-          end
-          sleep(SLEEP_SEC_BETWEEN_RETRY)
-          retry
-        end
-      end
-
-      def aspera_client_api_url
-        log_file = File.join(Dir.home, 'Library', 'Logs', 'IBM Aspera', 'ibm-aspera-desktop.log')
-        url = nil
-        File.open(log_file, 'r') do |file|
-          file.each_line do |line|
-            line = line.chomp
-            if (m = line.match(/JSON-RPC server listening on (.*)/))
-              url = "http://#{m[1]}"
-            end
-          end
-        end
-        return url
-      end
-
-      def start_transfer(transfer_spec, token_regenerator: nil)
-        @request_id = SecureRandom.uuid
-        # if there is a token, we ask connect client to use well known ssh private keys
-        # instead of asking password
-        transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
-        @client_app_api.start_transfer(app_id: @application_id,transfer_spec: transfer_spec)
-        # @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
-      end
-
-      def wait_for_transfers_completion
-        client_activity_args = {'aspera_client_settings' => @client_settings}
-        started = false
-        pre_calc = false
-        session_id = @xfer_id
-        begin
-          loop do
-            tr_info = @client_api.create("transfers/info/#{@xfer_id}", client_activity_args)[:data]
-            Log.log.trace1{Log.dump(:tr_info, tr_info)}
-            if tr_info['transfer_info'].is_a?(Hash)
-              transfer = tr_info['transfer_info']
-              if transfer.nil?
-                Log.log.warn('no session in Connect')
-                break
-              end
-              # TODO: get session id
-              case transfer['status']
-              when 'initiating', 'queued'
-                notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
-              when 'running'
-                if !started
-                  notify_progress(session_id: session_id, type: :session_start)
-                  started = true
-                end
-                if !pre_calc && (transfer['bytes_expected'] != 0)
-                  notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
-                  pre_calc = true
-                else
-                  notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
-                end
-              when 'completed'
-                notify_progress(type: :end, session_id: session_id)
-                break
-              when 'failed'
-                notify_progress(type: :end, session_id: session_id)
-                raise Fasp::Error, transfer['error_desc']
-              when 'cancelled'
-                notify_progress(type: :end, session_id: session_id)
-                raise Fasp::Error, 'Transfer cancelled by user'
-              else
-                notify_progress(type: :end, session_id: session_id)
-                raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
-              end
-            end
-            sleep(1)
-          end
-        rescue StandardError => e
-          return [e]
-        end
-        return [:success]
-      end # wait
-    end # AgentAspera
-  end
-end

data/lib/aspera/fasp/agent_base.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Aspera
-  module Fasp
-    # Base class for transfer agents
-    class AgentBase
-      class << self
-        # compute options from user provided and default options
-        def options(default:, options:)
-          result = options.symbolize_keys
-          available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
-          result.each do |k, _v|
-            raise "Unknown transfer agent parameter: #{k}, expect one of #{available}" unless default.key?(k)
-          end
-          default.each do |k, v|
-            raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
-            result[k] = v unless result.key?(k)
-          end
-          return result
-        end
-      end
-      def wait_for_completion
-        # list of: :success or "error message string"
-        statuses = wait_for_transfers_completion
-        @progress&.reset
-        raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
-        raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
-        return statuses
-      end
-
-      private
-
-      def initialize(options)
-        raise 'internal error' unless respond_to?(:start_transfer)
-        raise 'internal error' unless respond_to?(:wait_for_transfers_completion)
-        # method `shutdown` is optional
-        Log.log.debug{Log.dump(:agent_options, options)}
-        raise "transfer agent options expecting Hash, but have #{options.class}" unless options.is_a?(Hash)
-        @progress = options[:progress]
-        options.delete(:progress)
-      end
-
-      def notify_progress(**parameters)
-        @progress&.event(**parameters)
-      end
-    end
-  end
-end

data/lib/aspera/fasp/agent_trsdk.rb

@@ -1,146 +0,0 @@
-# frozen_string_literal: true
-
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/installation'
-require 'json'
-require 'uri'
-
-module Aspera
-  module Fasp
-    class AgentTrsdk < Aspera::Fasp::AgentBase
-      DEFAULT_OPTIONS = {
-        url:      'grpc://127.0.0.1:0',
-        external: false,
-        keep:     false
-      }.freeze
-      private_constant :DEFAULT_OPTIONS
-
-      # options come from transfer_info
-      def initialize(user_opts={})
-        super(user_opts)
-        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: user_opts)
-        daemon_uri = URI.parse(@options[:url])
-        raise Fasp::Error, "invalid url #{@options[:url]}" unless daemon_uri.scheme.eql?('grpc')
-        Log.log.debug{Log.dump(:agent_options, @options)}
-        # load and create SDK stub
-        $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
-        require 'transfer_services_pb'
-        # it stays
-        @daemon_pid = nil
-        begin
-          @transfer_client = Transfersdk::TransferService::Stub.new("#{daemon_uri.host}:#{daemon_uri.port}", :this_channel_is_insecure)
-          get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
-          Log.log.debug{"daemon info: #{get_info_response}"}
-          Log.log.warn{'attached to existing daemon'} unless @options[:external] || @options[:keep]
-          at_exit{shutdown}
-        rescue GRPC::Unavailable
-          raise if @options[:external]
-          raise "daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_uri}}" unless @daemon_pid.nil?
-          Log.log.warn('no daemon present, starting daemon...') if @options[:external]
-          # location of daemon binary
-          bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
-          # config file and logs are created in same folder
-          generated_config_file_path = File.join(bin_folder, 'sdk.conf')
-          log_base = File.join(bin_folder, 'transferd')
-          # create a config file for daemon
-          config = {
-            address:      daemon_uri.host,
-            port:         daemon_uri.port,
-            fasp_runtime: {
-              use_embedded: false,
-              user_defined: {
-                bin: bin_folder,
-                etc: bin_folder
-              }
-            }
-          }
-          File.write(generated_config_file_path, config.to_json)
-          @daemon_pid = Process.spawn(Installation.instance.path(:transferd), '--config', generated_config_file_path, out: "#{log_base}.out", err: "#{log_base}.err")
-          begin
-            # wait for process to initialize
-            Timeout.timeout(2.0) do
-              _, status = Process.wait2(@daemon_pid)
-              raise "transfer daemon exited with status #{status.exitstatus}. Check files: #{log_base}.out #{log_base}.err"
-            end
-          rescue Timeout::Error
-            nil
-          end
-          Log.log.debug{"daemon started with pid #{@daemon_pid}"}
-          Process.detach(@daemon_pid) if @options[:keep]
-          if daemon_uri.port.eql?(0)
-            # if port is zero, a dynamic port was created, get it
-            File.open("#{log_base}.out", 'r') do |file|
-              file.each_line do |line|
-                if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
-                  daemon_uri.port = m[2].to_i
-                  # no "break" , need to keep last one
-                end
-              end
-            end
-          end
-          retry
-        end
-      end
-
-      def start_transfer(transfer_spec, token_regenerator: nil)
-        # create a transfer request
-        transfer_request = Transfersdk::TransferRequest.new(
-          transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
-          config: Transfersdk::TransferConfig.new, # transfer configuration
-          transferSpec: transfer_spec.to_json) # transfer definition
-        # send start transfer request to the transfer manager daemon
-        start_transfer_response = @transfer_client.start_transfer(transfer_request)
-        Log.log.debug{"start transfer response #{start_transfer_response}"}
-        @transfer_id = start_transfer_response.transferId
-        Log.log.debug{"transfer started with id #{@transfer_id}"}
-      end
-
-      def wait_for_transfers_completion
-        # set to true when we know the total size of the transfer
-        session_started = false
-        bytes_expected = nil
-        # monitor transfer status
-        @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
-          Log.log.debug{Log.dump(:response, response.to_h)}
-          # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
-          case response.status
-          when :RUNNING
-            if !session_started
-              notify_progress(session_id: @transfer_id, type: :session_start)
-              session_started = true
-            end
-            if bytes_expected.nil? &&
-                !response.sessionInfo.preTransferBytes.eql?(0)
-              bytes_expected = response.sessionInfo.preTransferBytes
-              notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
-            end
-            notify_progress(type: :transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
-          when :COMPLETED
-            notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
-            notify_progress(type: :end, session_id: @transfer_id)
-            break
-          when :FAILED, :CANCELED
-            notify_progress(type: :end, session_id: @transfer_id)
-            raise Fasp::Error, JSON.parse(response.message)['Description']
-          when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
-            notify_progress(session_id: nil, type: :pre_start, info: response.status.to_s.downcase)
-          else
-            Log.log.error{"unknown status#{response.status}"}
-          end
-        end
-        # TODO: return status
-        return []
-      end
-
-      def shutdown
-        if !@options[:keep] && !@daemon_pid.nil?
-          Log.log.debug("stopping daemon #{@daemon_pid}")
-          Process.kill('INT', @daemon_pid)
-          _, status = Process.wait2(@daemon_pid)
-          Log.log.debug("daemon stopped #{status}")
-          @daemon_pid = nil
-        end
-      end
-    end
-  end
-end

data/lib/aspera/fasp/resume_policy.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-require 'singleton'
-require 'aspera/log'
-
-module Aspera
-  module Fasp
-    # implements a simple resume policy
-    class ResumePolicy
-      # list of supported parameters and default values
-      DEFAULTS = {
-        iter_max:      7,
-        sleep_initial: 2,
-        sleep_factor:  2,
-        sleep_max:     60
-      }.freeze
-
-      # @param params see DEFAULTS
-      def initialize(params=nil)
-        @parameters = DEFAULTS.dup
-        if !params.nil?
-          raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
-          params.each do |k, v|
-            raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
-            raise "#{k} must be Integer" unless v.is_a?(Integer)
-            @parameters[k] = v
-          end
-        end
-        Log.log.debug{"resume params=#{@parameters}"}
-      end
-
-      # calls block a number of times (resumes) until success or limit reached
-      # this is re-entrant, one resumer can handle multiple transfers in //
-      def execute_with_resume
-        raise 'block mandatory' unless block_given?
-        # maximum of retry
-        remaining_resumes = @parameters[:iter_max]
-        sleep_seconds = @parameters[:sleep_initial]
-        Log.log.debug{"retries=#{remaining_resumes}"}
-        # try to send the file until ascp is successful
-        loop do
-          Log.log.debug('transfer starting')
-          begin
-            # call provided block
-            yield
-            break
-          rescue Fasp::Error => e
-            Log.log.warn{"An error occurred: #{e.message}"}
-            # failure in ascp
-            if e.retryable?
-              # exit if we exceed the max number of retry
-              raise Fasp::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
-            else
-              # give one chance only to non retryable errors
-              unless remaining_resumes.eql?(@parameters[:iter_max])
-                Log.log.error('non-retryable error'.red.blink)
-                raise e
-              end
-            end
-          end
-
-          # take this retry in account
-          remaining_resumes -= 1
-          Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
-
-          # wait a bit before retrying, maybe network condition will be better
-          sleep(sleep_seconds)
-
-          # increase retry period
-          sleep_seconds *= @parameters[:sleep_factor]
-          # cap value
-          sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
-        end # loop
-      end
-    end
-  end
-end