aspera-cli 4.4.0 → 4.5.0

data/examples/faspex4.rb

@@ -1,11 +1,11 @@
 #!/usr/bin/env ruby
 # find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
 # this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
-# this example makes use of class Aspera::Fasp::Local for transfers, alternatively the official "Transfer SDK" could be used
+# this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
 # Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
 require 'aspera/rest'
 require 'aspera/log'
-require 'aspera/fasp/local'
+require 'aspera/fasp/agent_direct'
 
 tmpdir=ENV['tmp']||Dir.tmpdir || '.'
 
@@ -59,10 +59,8 @@ pkg_created=api_v3.create('send',package_create_params)[:data]
 transfer_spec=pkg_created['xfer_sessions'].first
 # set paths of files to send
 transfer_spec['paths']=[{'source'=>file_to_send}]
-# get the local agent (i.e. ascp)
-transfer_client=Aspera::Fasp::Local.new
-# disable ascp output on stdout (optional)
-transfer_client.quiet=true
+# get local agent (ascp), disable ascp output on stdout to not mix with JSON events
+transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
 # start transfer (asynchronous)
 job_id=transfer_client.start_transfer(transfer_spec)
 # wait for all transfer completion (for the example)

data/examples/transfer.rb

@@ -2,7 +2,7 @@
 # Example: transfer a file using one of the provided transfer agents
 # location of ascp can be specified with env var "ascp"
 # temp folder can be specified with env var "tmp"
-require 'aspera/fasp/local'
+require 'aspera/fasp/agent_direct'
 require 'aspera/fasp/listener'
 require 'aspera/fasp/installation'
 require 'aspera/log'
@@ -39,7 +39,7 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
 # 
 
 # get FASP Manager singleton based on above ascp location
-fasp_manager=Aspera::Fasp::Local.new
+fasp_manager=Aspera::Fasp::AgentDirect.new
 
 # Note that it would also be possible to start transfers using other agents
 #require 'aspera/fasp/connect'

data/lib/aspera/aoc.rb

@@ -2,7 +2,7 @@ require 'aspera/log'
 require 'aspera/rest'
 require 'aspera/hash_ext'
 require 'aspera/data_repository'
-require 'aspera/node'
+require 'aspera/fasp/default'
 require 'base64'
 
 module Aspera
@@ -18,19 +18,20 @@ module Aspera
     # to avoid infinite loop in pub link redirection
     MAX_REDIRECT=10
     CLIENT_APPS=['aspera.global-cli-client','aspera.drive']
+    # index offset in data repository of client app
     DATA_REPO_INDEX_START = 4
+    # cookie prefix so that console can decode identity
+    COOKIE_PREFIX='aspera.aoc'
 
     # path in URL of public links
-    PATHS_PUBLIC_LINK=['/packages/public/receive','/packages/public/send','/files/public']
+    PUBLIC_LINK_PATHS=['/packages/public/receive','/packages/public/send','/files/public']
     JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
     OAUTH_API_SUBPATH='api/v1/oauth2'
-    DEFAULT_TSPEC_INFO={
-      'remote_user' => Node::ACCESS_KEY_TRANSFER_USER,
-      'ssh_port'    => Node::SSH_PORT_DEFAULT,
-      'fasp_port'   => Node::UDP_PORT_DEFAULT
-    }
+    # minimum fields for user info if retrieval fails
+    USER_INFO_FIELDS_MIN=['name','email','id','default_workspace_id','organization_id']
 
-    private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:PATHS_PUBLIC_LINK,:JWT_AUDIENCE,:OAUTH_API_SUBPATH,:DEFAULT_TSPEC_INFO
+    private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE,
+    :OAUTH_API_SUBPATH,:COOKIE_PREFIX,:USER_INFO_FIELDS_MIN
 
     public
     # various API scopes supported
@@ -45,84 +46,119 @@ module Aspera
     FILES_APP='files'
     PACKAGES_APP='packages'
 
-    def self.get_client_info(client_name=CLIENT_APPS.first)
-      client_index=CLIENT_APPS.index(client_name)
-      raise "no such pre-defined client: #{client_name}" if client_index.nil?
+    # class static methods
+    class << self
       # strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
-      return client_name,Base64.urlsafe_encode64(DataRepository.instance.get_bin(DATA_REPO_INDEX_START+client_index))
-    end
+      def get_client_info(client_name=CLIENT_APPS.first)
+        client_index=CLIENT_APPS.index(client_name)
+        raise "no such pre-defined client: #{client_name}" if client_index.nil?
+        return client_name,Base64.urlsafe_encode64(DataRepository.instance.get_bin(DATA_REPO_INDEX_START+client_index))
+      end
 
-    # @param url of AoC instance
-    # @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
-    def self.parse_url(aoc_org_url)
-      uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
-      instance_fqdn=uri.host
-      Log.log.debug("instance_fqdn=#{instance_fqdn}")
-      raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
-      organization,instance_domain=instance_fqdn.split('.',2)
-      Log.log.debug("instance_domain=#{instance_domain}")
-      Log.log.debug("organization=#{organization}")
-      raise "expecting a public FQDN for #{PRODUCT_NAME}" if instance_domain.nil?
-      return organization,instance_domain
-    end
+      # @param url of AoC instance
+      # @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
+      def parse_url(aoc_org_url)
+        uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
+        instance_fqdn=uri.host
+        Log.log.debug("instance_fqdn=#{instance_fqdn}")
+        raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
+        organization,instance_domain=instance_fqdn.split('.',2)
+        Log.log.debug("instance_domain=#{instance_domain}")
+        Log.log.debug("organization=#{organization}")
+        raise "expecting a public FQDN for #{PRODUCT_NAME}" if instance_domain.nil?
+        return organization,instance_domain
+      end
 
-    # base API url depends on domain, which could be "qa.xxx"
-    def self.api_base_url(api_domain=PROD_DOMAIN)
-      return "https://api.#{api_domain}"
-    end
+      # base API url depends on domain, which could be "qa.xxx"
+      def api_base_url(api_domain=PROD_DOMAIN)
+        return "https://api.#{api_domain}"
+      end
 
-    def self.metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
-      return Rest.new({
-        :base_url => "#{api_base_url(api_domain)}/metering/v1",
-        :headers  => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
-      })
-    end
+      def metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
+        return Rest.new({
+          :base_url => "#{api_base_url(api_domain)}/metering/v1",
+          :headers  => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
+        })
+      end
 
-    # node API scopes
-    def self.node_scope(access_key,scope)
-      return 'node.'+access_key+':'+scope
-    end
+      # node API scopes
+      def node_scope(access_key,scope)
+        return 'node.'+access_key+':'+scope
+      end
 
-    def self.set_use_default_ports(val)
-      @@use_standard_ports=val
-    end
+      def set_use_default_ports(val)
+        @@use_standard_ports=val
+      end
 
-    # check option "link"
-    # if present try to get token value (resolve redirection if short links used)
-    # then set options url/token/auth
-    def self.resolve_pub_link(rest_opts,public_link_url)
-      return if public_link_url.nil?
-      # set to token if available after redirection
-      url_param_token_pair=nil
-      redirect_count=0
-      loop do
-        uri=URI.parse(public_link_url)
-        if PATHS_PUBLIC_LINK.include?(uri.path)
-          url_param_token_pair=URI::decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
-          if url_param_token_pair.nil?
-            raise ArgumentError,"link option must be URL with 'token' parameter"
+      # check option "link"
+      # if present try to get token value (resolve redirection if short links used)
+      # then set options url/token/auth
+      def resolve_pub_link(rest_opts,public_link_url)
+        return if public_link_url.nil?
+        # set to token if available after redirection
+        url_param_token_pair=nil
+        redirect_count=0
+        loop do
+          uri=URI.parse(public_link_url)
+          if PUBLIC_LINK_PATHS.include?(uri.path)
+            url_param_token_pair=URI::decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
+            if url_param_token_pair.nil?
+              raise ArgumentError,"link option must be URL with 'token' parameter"
+            end
+            # ok we get it !
+            rest_opts[:org_url]='https://'+uri.host
+            rest_opts[:auth][:grant]=:url_token
+            rest_opts[:auth][:url_token]=url_param_token_pair.last
+            return
           end
-          # ok we get it !
-          rest_opts[:org_url]='https://'+uri.host
-          rest_opts[:auth][:grant]=:url_token
-          rest_opts[:auth][:url_token]=url_param_token_pair.last
-          return
-        end
-        Log.log.debug("no expected format: #{public_link_url}")
-        raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
-        r = Net::HTTP.get_response(uri)
-        if r.code.start_with?("3")
-          public_link_url = r['location']
-          raise "no location in redirection" if public_link_url.nil?
-          Log.log.debug("redirect to: #{public_link_url}")
-        else
-          # not a redirection
-          raise ArgumentError,'link option must be redirect or have token parameter'
+          Log.log.debug("no expected format: #{public_link_url}")
+          raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
+          r = Net::HTTP.get_response(uri)
+          if r.code.start_with?("3")
+            public_link_url = r['location']
+            raise "no location in redirection" if public_link_url.nil?
+            Log.log.debug("redirect to: #{public_link_url}")
+          else
+            # not a redirection
+            raise ArgumentError,'link option must be redirect or have token parameter'
+          end
+        end # loop
+
+        raise RuntimeError,'too many redirections'
+      end
+
+      # additional transfer spec (tags) for package information
+      def package_tags(package_info,operation)
+        return {'tags'=>{'aspera'=>{'files'=>{
+          'package_id'        => package_info['id'],
+          'package_name'      => package_info['name'],
+          'package_operation' => operation
+          }}}}
+      end
+
+      # add details to show in analytics
+      def analytics_ts(app,direction,ws_id,ws_name)
+        # translate transfer to operation
+        operation=case direction
+        when 'send';    'upload'
+        when 'receive'; 'download'
+        else raise "ERROR: unexpected value: #{direction}"
         end
-      end # loop
 
-      raise RuntimeError,'too many redirections'
-    end
+        return {
+          'tags'        => {
+          'aspera'        => {
+          'usage_id'        => "aspera.files.workspace.#{ws_id}", # activity tracking
+          'files'           => {
+          'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
+          'workspace_name'        => ws_name,  # activity tracking
+          'workspace_id'          => ws_id,
+          }
+          }
+          }
+        }
+      end
+    end # static methods
 
     # @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
     def initialize(opt)
@@ -130,6 +166,7 @@ module Aspera
       # key: access key
       # value: associated secret
       @key_chain=nil
+      @user_info=nil
 
       # init rest params
       aoc_rest_p={:auth=>{:type =>:oauth2}}
@@ -197,50 +234,28 @@ module Aspera
 
     def key_chain=(keychain)
       raise "keychain already set" unless @key_chain.nil?
+      raise "keychain must have get_secret" unless keychain.respond_to?(:get_secret)
       @key_chain=keychain
       nil
     end
 
-    # additional transfer spec (tags) for package information
-    def self.package_tags(package_info,operation)
-      return {'tags'=>{'aspera'=>{'files'=>{
-        'package_id'        => package_info['id'],
-        'package_name'      => package_info['name'],
-        'package_operation' => operation
-        }}}}
-    end
-
-    # add details to show in analytics
-    def self.analytics_ts(app,direction,ws_id,ws_name)
-      # translate transfer to operation
-      operation=case direction
-      when 'send';    'upload'
-      when 'receive'; 'download'
-      else raise "ERROR: unexpected value: #{direction}"
+    # cached user information
+    def user_info
+      if @user_info.nil?
+        # get our user's default information
+        @user_info=self.read('self')[:data] rescue nil
+        @user_info=USER_INFO_FIELDS_MIN.inject({}){|m,f|m[f]=nil;m} if @user_info.nil?
+        USER_INFO_FIELDS_MIN.each{|f|@user_info[f]='unknown' if @user_info[f].nil?}
       end
-
-      return {
-        'tags'        => {
-        'aspera'        => {
-        'usage_id'        => "aspera.files.workspace.#{ws_id}", # activity tracking
-        'files'           => {
-        'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
-        'workspace_name'        => ws_name,  # activity tracking
-        'workspace_id'          => ws_id,
-        }
-        }
-        }
-      }
+      return @user_info
     end
 
     # build ts addon for IBM Aspera Console (cookie)
-    def self.console_ts(app,user_name,user_email)
-      elements=[app,user_name,user_email].map{|e|Base64.strict_encode64(e)}
-      elements.unshift('aspera.aoc')
-      #Log.dump('elem1'.bg_red,elements[1])
-      return {
-        'cookie'=>elements.join(':')
-      }
+    def console_ts(app)
+      # we are sure that fields are not nil
+      elements=[app,user_info['name'],user_info['email']].map{|e|Base64.strict_encode64(e)}
+      elements.unshift(COOKIE_PREFIX)
+      return {'cookie'=>elements.join(':')}
     end
 
     # build "transfer info", 2 elements array with:
@@ -248,12 +263,12 @@ module Aspera
     # - source and token regeneration method
     def tr_spec(app,direction,node_file,ts_add)
       # prepare the rest end point is used to generate the bearer token
-      token_generation_method=lambda {|do_refresh|self.oauth_token(scope: self.class.node_scope(node_file[:node_info]['access_key'],SCOPE_NODE_USER), refresh: do_refresh)}
+      token_generation_lambda=lambda {|do_refresh|self.oauth_token(scope: self.class.node_scope(node_file[:node_info]['access_key'],SCOPE_NODE_USER), refresh: do_refresh)}
       # prepare transfer specification
       # note xfer_id and xfer_retry are set by the transfer agent itself
       transfer_spec={
         'direction'   => direction,
-        'token'       => token_generation_method.call(false), # first time, use cache
+        'token'       => token_generation_lambda.call(false), # first time, use cache
         'tags'        => {
         'aspera'        => {
         'app'             => app,
@@ -270,8 +285,14 @@ module Aspera
       }
       # add remote host info
       if @@use_standard_ports
-        transfer_spec.merge!(DEFAULT_TSPEC_INFO)
+        # get default TCP/UDP ports and transfer user
+        transfer_spec.merge!(Fasp::Default::AK_TSPEC_BASE)
+        # by default: same address as node API
         transfer_spec['remote_host']=node_file[:node_info]['host']
+        # 30 it's necessarily https scheme: webui does not allow anything else
+        if node_file[:node_info]['transfer_url'].is_a?(String) and !node_file[:node_info]['transfer_url'].empty?
+          transfer_spec['remote_host']=URI.parse(node_file[:node_info]['transfer_url']).host
+        end
       else
         # retrieve values from API
         std_t_spec=get_node_api(node_file[:node_info],scope: SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
@@ -282,7 +303,7 @@ module Aspera
       # additional information for transfer agent
       source_and_token_generator={
         :src              => :node_gen4,
-        :regenerate_token => token_generation_method
+        :regenerate_token => token_generation_lambda
       }
       return transfer_spec,source_and_token_generator
     end
@@ -292,10 +313,10 @@ module Aspera
     # no scope: requires secret
     # if secret provided beforehand: use it
     def get_node_api(node_info,options={})
-      raise "INTERNAL ERROR: method parameters: options must ne hash" unless options.is_a?(Hash)
+      raise "INTERNAL ERROR: method parameters: options must be Hash" unless options.is_a?(Hash)
       options.keys.each {|k| raise "INTERNAL ERROR: not valid option: #{k}" unless [:scope,:use_secret].include?(k)}
       # get optional secret unless :use_secret is false (default is true)
-      ak_secret=@key_chain.get_secret(node_info['access_key'],false) if !options.has_key?(:use_secret) or options[:use_secret]
+      ak_secret=@key_chain.get_secret(url: node_info['url'], username: node_info['access_key'], mandatory: false) if !@key_chain.nil? and ( !options.has_key?(:use_secret) or options[:use_secret] )
       if ak_secret.nil? and !options.has_key?(:scope)
         raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
       end

data/lib/aspera/cli/listener/progress_multi.rb

@@ -1,5 +1,5 @@
 require 'aspera/fasp/listener'
-require 'aspera/fasp/manager'
+require 'aspera/fasp/agent_base'
 require 'ruby-progressbar'
 
 module Aspera
@@ -43,13 +43,13 @@ module Aspera
             :title      => '',
             :total      => nil)
           end
-          if !data.has_key?(Fasp::Manager::LISTENER_SESSION_ID_S)
-            Log.log.error("Internal error: no #{Fasp::Manager::LISTENER_SESSION_ID_S} in event: #{data}")
+          if !data.has_key?(Fasp::AgentBase::LISTENER_SESSION_ID_S)
+            Log.log.error("Internal error: no #{Fasp::AgentBase::LISTENER_SESSION_ID_S} in event: #{data}")
             return
           end
           newtitle=@sessions.length < 2 ? '' : "multi=#{@sessions.length}"
           @progress_bar.title=newtitle unless @progress_bar.title.eql?(newtitle)
-          session=@sessions[data[Fasp::Manager::LISTENER_SESSION_ID_S]]||={
+          session=@sessions[data[Fasp::AgentBase::LISTENER_SESSION_ID_S]]||={
             cumulative: 0,
             job_size: 0,
             current: 0
@@ -78,7 +78,7 @@ module Aspera
             # stop event when one file is completed
             session[:cumulative]=session[:cumulative]+data['size'].to_i
           when 'DONE' # end of session
-            @sessions.delete(data[Fasp::Manager::LISTENER_SESSION_ID_S])
+            @sessions.delete(data[Fasp::AgentBase::LISTENER_SESSION_ID_S])
             update_progress
             update_total
           else