aspera-cli 4.0.0.pre1

data/lib/aspera/faspex_gw.rb

@@ -0,0 +1,196 @@
+require 'aspera/log'
+require 'aspera/on_cloud'
+require 'aspera/cli/main'
+require 'webrick'
+require 'webrick/https'
+require 'securerandom'
+require 'openssl'
+require 'json'
+
+module Aspera
+  # this class answers the Faspex /send API and creates a package on Aspera on Cloud
+  class FaspexGW
+    class FxGwServlet < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server,a_aoc_api_user,a_workspace_id)
+        @aoc_api_user=a_aoc_api_user
+        @aoc_workspace_id=a_workspace_id
+      end
+
+      # parameters from user to Faspex API call
+      #{"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
+      #    {
+      #      "delivery"=>{
+      #      "use_encryption_at_rest"=>false,
+      #      "note"=>"note",
+      #      "sources"=>[{"paths"=>["file1"]}],
+      #      "title"=>"my title",
+      #      "recipients"=>["email1"],
+      #      "send_upload_result"=>true
+      #      }
+      #    }
+      def process_faspex_send(request, response)
+        raise "no payload" if request.body.nil?
+        faspex_pkg_parameters=JSON.parse(request.body)
+        faspex_pkg_delivery=faspex_pkg_parameters['delivery']
+        Log.log.debug "faspex pkg create parameters=#{faspex_pkg_parameters}"
+
+        # get recipient ids
+        files_pkg_recipients=[]
+        faspex_pkg_delivery['recipients'].each do |recipient_email|
+          user_lookup=@aoc_api_user.read("contacts",{'current_workspace_id'=>@aoc_workspace_id,'q'=>recipient_email})[:data]
+          raise StandardError,"no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? and user_lookup.length == 1
+          recipient_user_info=user_lookup.first
+          files_pkg_recipients.push({"id"=>recipient_user_info['source_id'],"type"=>recipient_user_info['source_type']})
+        end
+
+        #  create a new package with one file
+        the_package=@aoc_api_user.create("packages",{
+          "file_names"=>faspex_pkg_delivery['sources'][0]['paths'],
+          "name"=>faspex_pkg_delivery['title'],
+          "note"=>faspex_pkg_delivery['note'],
+          "recipients"=>files_pkg_recipients,
+          "workspace_id"=>@aoc_workspace_id})[:data]
+
+        #  get node information for the node on which package must be created
+        node_info=@aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
+
+        #  get transfer token (for node)
+        node_auth_bearer_token=@aoc_api_user.oauth_token(scope: OnCloud.node_scope(node_info['access_key'],OnCloud::SCOPE_NODE_USER))
+
+        # tell Files what to expect in package: 1 transfer (can also be done after transfer)
+        @aoc_api_user.update("packages/#{the_package['id']}",{"sent"=>true,"transfers_expected"=>1})
+
+        if false
+          response.status=400
+          return "ERROR HERE"
+        end
+        # TODO: check about xfer_*
+        ts_tags={
+          "aspera" => {
+          "files"      => { "package_id" => the_package['id'], "package_operation" => "upload" },
+          "node"       => { "access_key" => node_info['access_key'], "file_id" => the_package['contents_file_id'] },
+          "xfer_id"    => SecureRandom.uuid,
+          "xfer_retry" => 3600 } }
+        # this transfer spec is for transfer to AoC
+        faspex_transfer_spec={
+          'direction' => 'send',
+          'remote_user' => 'xfer',
+          'remote_host' => node_info['host'],
+          'ssh_port' => 33001,
+          'fasp_port' => 33001,
+          'tags' => ts_tags,
+          'token' => node_auth_bearer_token,
+          'paths' => [{'destination' => '/'}],
+          'cookie' => 'unused',
+          'create_dir' => true,
+          'rate_policy' => 'fair',
+          'rate_policy_allowed' => 'fixed',
+          'min_rate_cap_kbps' => nil,
+          'min_rate_kbps' => 0,
+          'target_rate_percentage' => nil,
+          'lock_target_rate' => nil,
+          'fasp_url' => 'unused',
+          'lock_min_rate' => true,
+          'lock_rate_policy' => true,
+          'source_root' => '',
+          'content_protection' => nil,
+          'target_rate_cap_kbps' => 20000, # TODO
+          'target_rate_kbps' => 10000, # TODO
+          'cipher' => 'aes-128',
+          'cipher_allowed' => nil,
+          'http_fallback' => false,
+          'http_fallback_port' => nil,
+          'https_fallback_port' => nil,
+          'destination_root' => '/'
+        }
+        # but we place it in a Faspex package creation response
+        faspex_package_create_result={
+          'links' => {'status' => 'unused'},
+          'xfer_sessions' => [faspex_transfer_spec]
+        }
+        Log.log.info("faspex_package_create_result=#{faspex_package_create_result}")
+        response.status=200
+        response.content_type = "application/json"
+        response.body=JSON.generate(faspex_package_create_result)
+      end
+
+      def do_GET (request, response)
+        case request.path
+        when '/aspera/faspex/send'
+          process_faspex_send(request, response)
+        else
+          response.status=400
+          return "ERROR HERE"
+          raise "unsupported path: #{request.path}"
+        end
+      end
+    end # FxGwServlet
+
+    class NewUserServlet < WEBrick::HTTPServlet::AbstractServlet
+      def do_GET (request, response)
+        case request.path
+        when '/newuser'
+          response.status=200
+          response.content_type = "text/html"
+          response.body='<html><body>hello world</body></html>'
+        else
+          raise "unsupported path: [#{request.path}]"
+        end
+      end
+    end
+
+    def fill_self_signed_cert(options)
+      key = OpenSSL::PKey::RSA.new(4096)
+      cert = OpenSSL::X509::Certificate.new
+      cert.subject = cert.issuer = OpenSSL::X509::Name.parse("/C=FR/O=Test/OU=Test/CN=Test")
+      cert.not_before = Time.now
+      cert.not_after = Time.now + 365 * 24 * 60 * 60
+      cert.public_key = key.public_key
+      cert.serial = 0x0
+      cert.version = 2
+      ef = OpenSSL::X509::ExtensionFactory.new
+      ef.issuer_certificate = cert
+      ef.subject_certificate = cert
+      cert.extensions = [
+        ef.create_extension("basicConstraints","CA:TRUE", true),
+        ef.create_extension("subjectKeyIdentifier", "hash"),
+        # ef.create_extension("keyUsage", "cRLSign,keyCertSign", true),
+      ]
+      cert.add_extension(ef.create_extension("authorityKeyIdentifier","keyid:always,issuer:always"))
+      cert.sign(key, OpenSSL::Digest::SHA256.new)
+      options[:SSLPrivateKey]  = key
+      options[:SSLCertificate] = cert
+    end
+
+    def initialize(a_aoc_api_user,a_workspace_id)
+      webrick_options = {
+        :app                => FaspexGW,
+        :Port               => 9443,
+        :Logger             => Log.log,
+        #:DocumentRoot       => Cli::Main.gem_root,
+        :SSLEnable          => true,
+        :SSLVerifyClient    => OpenSSL::SSL::VERIFY_NONE,
+      }
+      case 2
+      when 0
+        # generate self signed cert
+        webrick_options[:SSLCertName]    = [ [ 'CN',WEBrick::Utils::getservername ] ]
+        Log.log.error(">>>#{webrick_options[:SSLCertName]}")
+      when 1
+        fill_self_signed_cert(webrick_options)
+      when 2
+        webrick_options[:SSLPrivateKey] =OpenSSL::PKey::RSA.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.key'))
+        webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.crt'))
+      end
+      Log.log.info("Server started on port #{webrick_options[:Port]}")
+      @server = WEBrick::HTTPServer.new(webrick_options)
+      @server.mount('/aspera/faspex', FxGwServlet,a_aoc_api_user,a_workspace_id)
+      @server.mount('/newuser', NewUserServlet)
+      trap('INT') {@server.shutdown}
+    end
+
+    def start_server
+      @server.start
+    end
+  end # FaspexGW
+end # AsperaLm

data/lib/aspera/hash_ext.rb

@@ -0,0 +1,28 @@
+# for older rubies
+unless Hash.method_defined?(:dig)
+  class Hash
+    def dig(*path)
+      path.inject(self) do |location, key|
+        location.respond_to?(:keys) ? location[key] : nil
+      end
+    end
+  end
+end
+
+class ::Hash
+  def deep_merge(second)
+    self.merge(second){|key,v1,v2|Hash===v1&&Hash===v2 ? v1.deep_merge(v2) : v2}
+  end
+
+  def deep_merge!(second)
+    self.merge!(second){|key,v1,v2|Hash===v1&&Hash===v2 ? v1.deep_merge!(v2) : v2}
+  end
+end
+
+unless Hash.method_defined?(:symbolize_keys)
+  class Hash
+    def symbolize_keys
+      return self.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+    end
+  end
+end

data/lib/aspera/log.rb

@@ -0,0 +1,80 @@
+require 'aspera/colors'
+require 'logger'
+require 'pp'
+require 'json'
+require 'singleton'
+
+module Aspera
+  # Singleton object for logging
+  class Log
+
+    public
+    include Singleton
+
+    attr_reader :logger
+    attr_reader :logger_type
+    # levels are :debug,:info,:warn,:error,fatal,:unknown
+    def self.levels; Logger::Severity.constants.map{|c| c.downcase.to_sym};end
+
+    # where logs are sent to
+    def self.logtypes; [:stderr,:stdout,:syslog];end
+
+    # get the logger object of singleton
+    def self.log; self.instance.logger; end
+
+    # dump object in debug mode
+    # @param name string or symbol
+    # @param format either pp or json format
+    def self.dump(name,object,format=:json)
+      result=case format
+      when :ruby;PP.pp(object,'')
+      when :json;JSON.pretty_generate(object) rescue PP.pp(object,'')
+      else raise "wrong parameter, expect pp or json"
+      end
+      self.log.debug("#{name.to_s.green} (#{format})=\n#{result}")
+    end
+
+    # set log level of underlying logger given symbol level
+    def level=(new_level)
+      @logger.level=Logger::Severity.const_get(new_level.to_sym.upcase)
+    end
+
+    # get symbol of debug level of underlying logger
+    def level
+      Logger::Severity.constants.each do |name|
+        return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
+      end
+      raise "error"
+    end
+
+    # change underlying logger, but keep log level
+    def logger_type=(new_logtype)
+      current_severity_integer=@logger.nil? ? Logger::Severity::WARN : @logger.level
+      case new_logtype
+      when :stderr
+        @logger = Logger.new(STDERR)
+      when :stdout
+        @logger = Logger.new(STDOUT)
+      when :syslog
+        require 'syslog/logger'
+        @logger = Syslog::Logger.new(@program_name)
+      else
+        raise "unknown log type: #{new_logtype.class} #{new_logtype}"
+      end
+      @logger.level=current_severity_integer
+      @logger_type=new_logtype
+    end
+
+    attr_writer :program_name
+
+    private
+
+    def initialize
+      @logger=nil
+      @program_name='aspera'
+      # this sets @logger and @logger_type
+      self.logger_type=:stderr
+    end
+
+  end
+end

data/lib/aspera/nagios.rb

@@ -0,0 +1,71 @@
+require 'date'
+
+module Aspera
+  class Nagios
+    # nagios levels
+    LEVELS=[:ok,:warning,:critical,:unknown,:dependent]
+    ADD_PREFIX='add_'
+    # add methods to add nagios error levels, each take component name and message
+    LEVELS.each_index do |code|
+      name="#{ADD_PREFIX}#{LEVELS[code]}".to_sym
+      define_method(name){|comp,msg|@data.push({:code=>code,:comp=>comp,:msg=>msg})}
+      public name
+    end
+    # date offset levels
+    DATE_WARN_OFFSET=2
+    DATE_CRIT_OFFSET=5
+    private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
+
+    attr_reader :data
+    def initialize
+      @data=[]
+    end
+
+    # comparte remote time with local time
+    def check_time_offset( remote_date, component )
+      # check date if specified : 2015-10-13T07:32:01Z
+      rtime = DateTime.strptime(remote_date)
+      diff_time = (rtime - DateTime.now).abs
+      diff_disp=diff_time.round(-2)
+      Log.log.debug("DATE: #{remote_date} #{rtime} diff=#{diff_disp}")
+      msg="offset #{diff_disp} sec"
+      if diff_time >= DATE_CRIT_OFFSET
+        add_critical(component,msg)
+      elsif diff_time >= DATE_WARN_OFFSET
+        add_warning(component,msg)
+      else
+        add_ok(component,msg)
+      end
+    end
+
+    def check_product_version( component, product, version )
+      add_ok(component,"version #{version}")
+      # TODO check on database if latest version
+    end
+
+    # translate for display
+    def result
+      raise "missing result" if @data.empty?
+      {:type=>:object_list,:data=>@data.map{|i|{'status'=>LEVELS[i[:code]].to_s,'component'=>i[:comp],'message'=>i[:msg]}}}
+    end
+
+    # process results of a analysis and display status and exit with code
+    def self.process(data)
+      raise "INTERNAL ERROR, result must be list and not empty" unless data.is_a?(Array) and !data.empty?
+      ['status','component','message'].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
+      res_errors = data.select{|s|!s['status'].eql?('ok')}
+      # keep only errors in case of problem, other ok are assumed so
+      data = res_errors unless res_errors.empty?
+      # first is most critical
+      data.sort!{|a,b|LEVELS.index(a['status'].to_sym)<=>LEVELS.index(b['status'].to_sym)}
+      # build message: if multiple components: concatenate
+      #message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
+      message = data.map{|i|i['component']}.uniq.map{|comp|comp+':'+data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').gsub("\n",' ')
+      status=data.first['status'].upcase
+      # display status for nagios
+      puts("#{status} - [#{message}]\n")
+      # provide exit code to nagios
+      Process.exit(LEVELS.index(data.first['status'].to_sym))
+    end
+  end
+end

data/lib/aspera/node.rb

@@ -0,0 +1,14 @@
+require 'zlib'
+require 'base64'
+module Aspera
+  # Provides additional functions using node API.
+  class Node < Rest
+    def self.decode_bearer_token(token)
+      return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
+    end
+    def initialize(rest_params)
+      super(rest_params)
+      # specifics here
+    end
+  end
+end

data/lib/aspera/oauth.rb

@@ -0,0 +1,319 @@
+require 'aspera/open_application'
+require 'base64'
+require 'date'
+require 'socket'
+require 'securerandom'
+
+module Aspera
+  # implement OAuth 2 for the REST client and generate a bearer token
+  # call get_authorization() to get a token.
+  # bearer tokens are kept in memory and also in a file cache for later re-use
+  # if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
+  class Oauth
+    private
+    # remove 5 minutes to account for time offset (TODO: configurable?)
+    JWT_NOTBEFORE_OFFSET=300
+    # one hour validity (TODO: configurable?)
+    JWT_EXPIRY_OFFSET=3600
+    PERSIST_CATEGORY_TOKEN='token'
+    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN
+    class << self
+      # OAuth methods supported
+      def auth_types
+        [ :body_userpass, :header_userpass, :web, :jwt, :url_token, :ibm_apikey ]
+      end
+
+      def persist_mgr=(manager)
+        @persist=manager
+      end
+
+      def persist_mgr
+        raise "set persistency manager first" if @persist.nil?
+        return @persist
+      end
+
+      def flush_tokens
+        persist_mgr.flush_by_prefix(PERSIST_CATEGORY_TOKEN)
+      end
+    end
+
+    # for supported parameters, look in the code for @params
+    # parameters are provided all with oauth_ prefix :
+    # :base_url
+    # :client_id
+    # :client_secret
+    # :redirect_uri
+    # :jwt_audience
+    # :jwt_private_key_obj
+    # :jwt_subject
+    # :path_authorize (default: 'authorize')
+    # :path_token (default: 'token')
+    # :scope (optional)
+    # :grant (one of returned by self.auth_types)
+    # :url_token
+    # :user_name
+    # :user_pass
+    # :token_type
+    def initialize(auth_params)
+      Log.log.debug "auth=#{auth_params}"
+      @params=auth_params.clone
+      # default values
+      # name of field to take as token from result of call to /token
+      @params[:token_field]||='access_token'
+      # default endpoint for /token
+      @params[:path_token]||='token'
+      # default endpoint for /authorize
+      @params[:path_authorize]||='authorize'
+      rest_params={:base_url => @params[:base_url]}
+      if @params.has_key?(:client_id)
+        rest_params.merge!({:auth     => {
+          :type     => :basic,
+          :username => @params[:client_id],
+          :password => @params[:client_secret]
+          }})
+      end
+      @token_auth_api=Rest.new(rest_params)
+      if @params.has_key?(:redirect_uri)
+        uri=URI.parse(@params[:redirect_uri])
+        raise "redirect_uri scheme must be http" unless uri.scheme.start_with?('http')
+        raise "redirect_uri must have a port" if uri.port.nil?
+        # we could check that host is localhost or local address
+      end
+    end
+
+    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
+
+    # open the login page, wait for code and check_code, then return code
+    def goto_page_and_get_code(login_page_url,check_code)
+      request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
+      Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
+      code=request_params['code']
+      return code
+    end
+
+    def create_token_advanced(rest_params)
+      return @token_auth_api.call({
+        :operation => 'POST',
+        :subpath   => @params[:path_token],
+        :headers   => {'Accept'=>'application/json'}}.merge(rest_params))
+    end
+
+    # shortcut for create_token_advanced
+    def create_token_www_body(creation_params)
+      return create_token_advanced({:www_body_params=>creation_params})
+    end
+
+    # @return Array list of unique identifiers of token
+    def token_cache_ids(api_scope)
+      oauth_uri=URI.parse(@params[:base_url])
+      parts=[PERSIST_CATEGORY_TOKEN,oauth_uri.host.downcase.gsub(/[^a-z]+/,'_'),oauth_uri.path.downcase.gsub(/[^a-z]+/,'_'),@params[:grant]]
+      parts.push(api_scope) unless api_scope.nil?
+      parts.push(@params[:jwt_subject]) if @params.has_key?(:jwt_subject)
+      parts.push(@params[:user_name]) if @params.has_key?(:user_name)
+      parts.push(@params[:url_token]) if @params.has_key?(:url_token)
+      parts.push(@params[:api_key]) if @params.has_key?(:api_key)
+      return parts
+    end
+
+    public
+
+    # @param options : :scope and :refresh
+    def get_authorization(options={})
+      # api scope can be overriden to get auth for other scope
+      api_scope=options[:scope] || @params[:scope]
+      # as it is optional in many place: create struct
+      p_scope={}
+      p_scope[:scope] = api_scope unless api_scope.nil?
+      p_client_id_and_scope=p_scope.clone
+      p_client_id_and_scope[:client_id] = @params[:client_id] if @params.has_key?(:client_id)
+      use_refresh_token=options[:refresh]
+
+      # generate token identifier to use with cache
+      token_ids=token_cache_ids(api_scope)
+
+      # get token_data from cache (or nil), token_data is what is returned by /token
+      token_data=self.class.persist_mgr.get(token_ids)
+      token_data=JSON.parse(token_data) unless token_data.nil?
+
+      # Optional optimization: check if node token is expired, then force refresh
+      # in case the transfer agent cannot refresh himself
+      # else, anyway, faspmanager is equipped with refresh code
+      if !token_data.nil?
+        decoded_node_token = Node.decode_bearer_token(token_data['access_token']) rescue nil
+        if decoded_node_token.is_a?(Hash) and decoded_node_token['expires_at'].is_a?(String)
+          Log.dump('decoded_node_token',decoded_node_token)
+          expires_at=DateTime.parse(decoded_node_token['expires_at'])
+          one_hour_as_day_fraction=Rational(1,24)
+          use_refresh_token=true if DateTime.now > (expires_at-one_hour_as_day_fraction)
+        end
+      end
+
+      # an API was already called, but failed, we need to regenerate or refresh
+      if use_refresh_token
+        if token_data.is_a?(Hash) and token_data.has_key?('refresh_token')
+          # save possible refresh token, before deleting the cache
+          refresh_token=token_data['refresh_token']
+        end
+        # delete caches
+        self.class.persist_mgr.delete(token_ids)
+        token_data=nil
+        # lets try the existing refresh token
+        if !refresh_token.nil?
+          Log.log.info("refresh=[#{refresh_token}]".bg_green)
+          # try to refresh
+          # note: admin token has no refresh, and lives by default 1800secs
+          # Note: scope is mandatory in Files, and we can either provide basic auth, or client_Secret in data
+          resp=create_token_www_body(p_client_id_and_scope.merge({
+            :grant_type   =>'refresh_token',
+            :refresh_token=>refresh_token}))
+          if resp[:http].code.start_with?('2') then
+            # save only if success ?
+            json_data=resp[:http].body
+            token_data=JSON.parse(json_data)
+            self.class.persist_mgr.put(token_ids,json_data)
+          else
+            Log.log.debug("refresh failed: #{resp[:http].body}".bg_red)
+          end
+        end
+      end
+
+      # no cache
+      if token_data.nil? then
+        resp=nil
+        case @params[:grant]
+        when :web
+          # AoC Web based Auth
+          check_code=SecureRandom.uuid
+          login_page_url=Rest.build_uri(
+          "#{@params[:base_url]}/#{@params[:path_authorize]}",
+          p_client_id_and_scope.merge({
+            :response_type => 'code',
+            :redirect_uri  => @params[:redirect_uri],
+            :client_secret => @params[:client_secret],
+            :state         => check_code
+          }))
+          # here, we need a human to authorize on a web page
+          code=goto_page_and_get_code(login_page_url,check_code)
+          # exchange code for token
+          resp=create_token_www_body(p_client_id_and_scope.merge({
+            :grant_type   => 'authorization_code',
+            :code         => code,
+            :redirect_uri => @params[:redirect_uri]
+          }))
+        when :jwt
+          # https://tools.ietf.org/html/rfc7519
+          # https://tools.ietf.org/html/rfc7523
+          require 'jwt'
+          seconds_since_epoch=Time.new.to_i
+          Log.log.info("seconds=#{seconds_since_epoch}")
+
+          payload = {
+            :iss => @params[:client_id],    # issuer
+            :sub => @params[:jwt_subject],  # subject
+            :aud => @params[:jwt_audience], # audience
+            :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
+            :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
+          }
+
+          # non standard, only for global ids
+          payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
+
+          rsa_private=@params[:jwt_private_key_obj]  # type: OpenSSL::PKey::RSA
+
+          Log.log.debug("private=[#{rsa_private}]")
+
+          Log.log.debug("JWT assertion=[#{payload}]")
+          assertion = JWT.encode(payload, rsa_private, 'RS256')
+
+          Log.log.debug("assertion=[#{assertion}]")
+
+          resp=create_token_www_body(p_scope.merge({
+            :grant_type => 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+            :assertion  => assertion
+          }))
+        when :url_token
+          # AoC Public Link
+          resp=create_token_advanced({
+            :json_params => {:url_token=>@params[:url_token]},
+            :url_params  => p_scope.merge({
+            :grant_type    => 'url_token'
+            })})
+        when :ibm_apikey
+          # ATS
+          resp=create_token_www_body({
+            'grant_type'    => 'urn:ibm:params:oauth:grant-type:apikey',
+            'response_type' => 'cloud_iam',
+            'apikey'        => @params[:api_key]
+          })
+        when :delegated_refresh
+          # COS
+          resp=create_token_www_body({
+            'grant_type'          => 'urn:ibm:params:oauth:grant-type:apikey',
+            'response_type'       => 'delegated_refresh_token',
+            'apikey'              => @params[:api_key],
+            'receiver_client_ids' => 'aspera_ats'
+          })
+        when :header_userpass
+          # used in Faspex apiv4 and shares2
+          resp=create_token_advanced({
+            :auth        => {
+            :type          => :basic,
+            :username      => @params[:user_name],
+            :password      => @params[:user_pass]},
+            :json_params => p_client_id_and_scope.merge({:grant_type => 'password'}), #:www_body_params also works
+          })
+        when :body_userpass
+          # legacy, not used
+          resp=create_token_www_body(p_client_id_and_scope.merge({
+            :grant_type => 'password',
+            :username   => @params[:user_name],
+            :password   => @params[:user_pass]
+          }))
+        when :body_data
+          # used in Faspex apiv5
+          resp=create_token_advanced({
+            :auth        => {:type => :none},
+            :json_params => @params[:userpass_body],
+          })
+        else
+          raise "auth grant type unknown: #{@params[:grant]}"
+        end
+        # TODO: test return code ?
+        json_data=resp[:http].body
+        token_data=JSON.parse(json_data)
+        self.class.persist_mgr.put(token_ids,json_data)
+      end # if ! in_cache
+
+      # ok we shall have a token here
+      return 'Bearer '+token_data[@params[:token_field]]
+    end
+
+    # open the login page, wait for code and return parameters
+    def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
+      Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
+      # browser start is not blocking, we hope here that starting is slower than opening port
+      OpenApplication.instance.uri(login_page_url)
+      port=URI.parse(redirect_uri).port
+      Log.log.info "listening on port #{port}"
+      request_params=nil
+      TCPServer.open('127.0.0.1', port) { |webserver|
+        Log.log.info "server=#{webserver}"
+        websession = webserver.accept
+        sleep 1 # TODO: sometimes: returns nil ? use webrick ?
+        line = websession.gets.chomp
+        Log.log.info "line=#{line}"
+        if ! line.start_with?('GET /?') then
+          raise "unexpected request"
+        end
+        request = line.partition('?').last.partition(' ').first
+        data=URI.decode_www_form(request)
+        request_params=data.to_h
+        Log.log.debug "request_params=#{request_params}"
+        websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
+        websession.close
+      }
+      return request_params
+    end
+
+  end # OAuth
+end # Aspera