aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/oauth.rb

@@ -1,5 +1,6 @@
 require 'aspera/open_application'
 require 'aspera/web_auth'
+require 'aspera/id_generator'
 require 'base64'
 require 'date'
 require 'socket'
@@ -9,8 +10,14 @@ module Aspera
   # Implement OAuth 2 for the REST client and generate a bearer token
   # call get_authorization() to get a token.
   # bearer tokens are kept in memory and also in a file cache for later re-use
-  # if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
+  # if a token is expired (api returns 4xx), call again get_authorization({refresh: true})
   class Oauth
+    # used for code exchange
+    DEFAULT_PATH_AUTHORIZE='authorize'
+    # to generate token
+    DEFAULT_PATH_TOKEN='token'
+    # field with token in result
+    DEFAULT_TOKEN_FIELD='access_token'
     private
     # remove 5 minutes to account for time offset (TODO: configurable?)
     JWT_NOTBEFORE_OFFSET_SEC=300
@@ -20,7 +27,9 @@ module Aspera
     TOKEN_CACHE_EXPIRY_SEC=1800
     # a prefix for persistency of tokens (garbage collect)
     PERSIST_CATEGORY_TOKEN='token'
-    private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC
+    ONE_HOUR_AS_DAY_FRACTION=Rational(1,24)
+
+    private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC,:ONE_HOUR_AS_DAY_FRACTION
     class << self
       # OAuth methods supported
       def auth_types
@@ -45,8 +54,25 @@ module Aspera
       def flush_tokens
         persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
       end
+
+      def register_decoder(method)
+        @decoders||=[]
+        @decoders.push(method)
+      end
+
+      def decode_token(token)
+        Log.log.debug(">>>> #{token} : #{@decoders.length}")
+        @decoders.each do |decoder|
+          result=decoder.call(token) rescue nil
+          return result unless result.nil?
+        end
+        return nil
+      end
     end
 
+    # seems to be quite standard token encoding (RFC?)
+    self.register_decoder lambda { |token| parts=token.split('.'); raise "not aoc token" unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
+
     # for supported parameters, look in the code for @params
     # parameters are provided all with oauth_ prefix :
     # :base_url
@@ -56,8 +82,8 @@ module Aspera
     # :jwt_audience
     # :jwt_private_key_obj
     # :jwt_subject
-    # :path_authorize (default: 'authorize')
-    # :path_token (default: 'token')
+    # :path_authorize (default: DEFAULT_PATH_AUTHORIZE)
+    # :path_token (default: DEFAULT_PATH_TOKEN)
     # :scope (optional)
     # :grant (one of returned by self.auth_types)
     # :url_token
@@ -65,21 +91,21 @@ module Aspera
     # :user_pass
     # :token_type
     def initialize(auth_params)
-      Log.log.debug "auth=#{auth_params}"
+      Log.log.debug("auth=#{auth_params}")
       @params=auth_params.clone
       # default values
       # name of field to take as token from result of call to /token
-      @params[:token_field]||='access_token'
+      @params[:token_field]||=DEFAULT_TOKEN_FIELD
       # default endpoint for /token
-      @params[:path_token]||='token'
+      @params[:path_token]||=DEFAULT_PATH_TOKEN
       # default endpoint for /authorize
-      @params[:path_authorize]||='authorize'
-      rest_params={:base_url => @params[:base_url]}
+      @params[:path_authorize]||=DEFAULT_PATH_AUTHORIZE
+      rest_params={base_url: @params[:base_url]}
       if @params.has_key?(:client_id)
-        rest_params.merge!({:auth     => {
-          :type     => :basic,
-          :username => @params[:client_id],
-          :password => @params[:client_secret]
+        rest_params.merge!({auth: {
+          type:     :basic,
+          username: @params[:client_id],
+          password: @params[:client_secret]
           }})
       end
       @token_auth_api=Rest.new(rest_params)
@@ -107,28 +133,20 @@ module Aspera
       return code
     end
 
-    def create_token_advanced(rest_params)
+    def create_token(rest_params)
       return @token_auth_api.call({
-        :operation => 'POST',
-        :subpath   => @params[:path_token],
-        :headers   => {'Accept'=>'application/json'}}.merge(rest_params))
-    end
-
-    # shortcut for create_token_advanced
-    def create_token_www_body(creation_params)
-      return create_token_advanced({:www_body_params=>creation_params})
+        operation: 'POST',
+        subpath:   @params[:path_token],
+        headers:   {'Accept'=>'application/json'}}.merge(rest_params))
     end
 
-    # @return Array list of unique identifiers of token
-    def token_cache_ids(api_scope)
+    # @return unique identifier of token
+    def token_cache_id(api_scope)
       oauth_uri=URI.parse(@params[:base_url])
-      parts=[PERSIST_CATEGORY_TOKEN,oauth_uri.host.downcase.gsub(/[^a-z]+/,'_'),oauth_uri.path.downcase.gsub(/[^a-z]+/,'_'),@params[:grant]]
-      parts.push(api_scope) unless api_scope.nil?
-      parts.push(@params[:jwt_subject]) if @params.has_key?(:jwt_subject)
-      parts.push(@params[:user_name]) if @params.has_key?(:user_name)
-      parts.push(@params[:url_token]) if @params.has_key?(:url_token)
-      parts.push(@params[:api_key]) if @params.has_key?(:api_key)
-      return parts
+      parts=[PERSIST_CATEGORY_TOKEN,api_scope,oauth_uri.host,oauth_uri.path]
+      # add some of the parameters that uniquely define the token
+      [:grant,:jwt_subject,:user_name,:url_token,:api_key].inject(parts){|p,i|p.push(@params[i])}
+      return IdGenerator.from_list(parts)
     end
 
     public
@@ -148,22 +166,23 @@ module Aspera
       use_refresh_token=options[:refresh]
 
       # generate token identifier to use with cache
-      token_ids=token_cache_ids(api_scope)
+      token_id=token_cache_id(api_scope)
 
       # get token_data from cache (or nil), token_data is what is returned by /token
-      token_data=self.class.persist_mgr.get(token_ids)
+      token_data=self.class.persist_mgr.get(token_id)
       token_data=JSON.parse(token_data) unless token_data.nil?
-
       # Optional optimization: check if node token is expired, then force refresh
       # in case the transfer agent cannot refresh himself
       # else, anyway, faspmanager is equipped with refresh code
       if !token_data.nil?
-        decoded_node_token = Node.decode_bearer_token(token_data['access_token']) rescue nil
+        # TODO: use @params[:token_field] ?
+        decoded_node_token = self.class.decode_token(token_data['access_token'])
+        Log.dump('decoded_node_token',decoded_node_token) unless decoded_node_token.nil?
         if decoded_node_token.is_a?(Hash) and decoded_node_token['expires_at'].is_a?(String)
-          Log.dump('decoded_node_token',decoded_node_token)
           expires_at=DateTime.parse(decoded_node_token['expires_at'])
-          one_hour_as_day_fraction=Rational(1,24)
-          use_refresh_token=true if DateTime.now > (expires_at-one_hour_as_day_fraction)
+          # Time.at(decoded_node_token['exp'])
+          # does it seem expired, with one hour of security
+          use_refresh_token=true if DateTime.now > (expires_at-ONE_HOUR_AS_DAY_FRACTION)
         end
       end
 
@@ -174,7 +193,7 @@ module Aspera
           refresh_token=token_data['refresh_token']
         end
         # delete caches
-        self.class.persist_mgr.delete(token_ids)
+        self.class.persist_mgr.delete(token_id)
         token_data=nil
         # lets try the existing refresh token
         if !refresh_token.nil?
@@ -182,14 +201,14 @@ module Aspera
           # try to refresh
           # note: admin token has no refresh, and lives by default 1800secs
           # Note: scope is mandatory in Files, and we can either provide basic auth, or client_Secret in data
-          resp=create_token_www_body(p_client_id_and_scope.merge({
-            :grant_type   =>'refresh_token',
-            :refresh_token=>refresh_token}))
+          resp=create_token(www_body_params: p_client_id_and_scope.merge({
+            grant_type:    'refresh_token',
+            refresh_token: refresh_token}))
           if resp[:http].code.start_with?('2') then
-            # save only if success ?
+            # save only if success
             json_data=resp[:http].body
             token_data=JSON.parse(json_data)
-            self.class.persist_mgr.put(token_ids,json_data)
+            self.class.persist_mgr.put(token_id,json_data)
           else
             Log.log.debug("refresh failed: #{resp[:http].body}".bg_red)
           end
@@ -204,19 +223,19 @@ module Aspera
           # AoC Web based Auth
           check_code=SecureRandom.uuid
           auth_params=p_client_id_and_scope.merge({
-            :response_type => 'code',
-            :redirect_uri  => @params[:redirect_uri],
-            :state         => check_code
+            response_type: 'code',
+            redirect_uri:  @params[:redirect_uri],
+            state:         check_code
           })
           auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
           login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
           # here, we need a human to authorize on a web page
           code=goto_page_and_get_code(login_page_url,check_code)
           # exchange code for token
-          resp=create_token_www_body(p_client_id_and_scope.merge({
-            :grant_type   => 'authorization_code',
-            :code         => code,
-            :redirect_uri => @params[:redirect_uri]
+          resp=create_token(www_body_params: p_client_id_and_scope.merge({
+            grant_type:   'authorization_code',
+            code:         code,
+            redirect_uri: @params[:redirect_uri]
           }))
         when :jwt
           # https://tools.ietf.org/html/rfc7519
@@ -226,84 +245,81 @@ module Aspera
           Log.log.info("seconds=#{seconds_since_epoch}")
 
           payload = {
-            :iss => @params[:client_id],    # issuer
-            :sub => @params[:jwt_subject],  # subject
-            :aud => @params[:jwt_audience], # audience
-            :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
-            :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
+            iss: @params[:client_id],    # issuer
+            sub: @params[:jwt_subject],  # subject
+            aud: @params[:jwt_audience], # audience
+            nbf: seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
+            exp: seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
           }
           # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
-          if @params[:jwt_is_f5]
-            payload[:jti] = SecureRandom.uuid
-            payload[:iat] = seconds_since_epoch
-            payload.delete(:nbf)
-            p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
+          if @params.has_key?(:f5_username)
+            payload[:jti] = SecureRandom.uuid # JWT id
+            payload[:iat] = seconds_since_epoch # issued at
+            payload.delete(:nbf) # not used in f5
+            p_scope[:redirect_uri]="https://127.0.0.1:5000/token" # used ?
             p_scope[:state]=SecureRandom.uuid
             p_scope[:client_id]=@params[:client_id]
-            @token_auth_api.params[:auth]={:type=>:none}
+            @token_auth_api.params[:auth]={type: :basic,username: @params[:f5_username], password: @params[:f5_password]}
           end
 
           # non standard, only for global ids
           payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
+          Log.log.debug("JWT payload=[#{payload}]")
 
           rsa_private=@params[:jwt_private_key_obj]  # type: OpenSSL::PKey::RSA
-
           Log.log.debug("private=[#{rsa_private}]")
 
-          Log.log.debug("JWT payload=[#{payload}]")
-          assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
-
+          assertion = JWT.encode(payload, rsa_private, 'RS256', @params[:jwt_headers]||{})
           Log.log.debug("assertion=[#{assertion}]")
 
-          resp=create_token_www_body(p_scope.merge({
-            :grant_type => 'urn:ietf:params:oauth:grant-type:jwt-bearer',
-            :assertion  => assertion
+          resp=create_token(www_body_params: p_scope.merge({
+            grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+            assertion:  assertion
           }))
         when :url_token
           # AoC Public Link
-          params={:url_token=>@params[:url_token]}
+          params={url_token: @params[:url_token]}
           params[:password]=@params[:password] if @params.has_key?(:password)
-          resp=create_token_advanced({
-            :json_params => params,
-            :url_params  => p_scope.merge({
-            :grant_type    => 'url_token'
-            })})
+          resp=create_token({
+            json_params: params,
+            url_params:  p_scope.merge({grant_type: 'url_token'})
+          })
         when :ibm_apikey
           # ATS
-          resp=create_token_www_body({
-            'grant_type'    => 'urn:ibm:params:oauth:grant-type:apikey',
-            'response_type' => 'cloud_iam',
-            'apikey'        => @params[:api_key]
+          resp=create_token(www_body_params: {
+            grant_type:    'urn:ibm:params:oauth:grant-type:apikey',
+            response_type: 'cloud_iam',
+            apikey:        @params[:api_key]
           })
         when :delegated_refresh
           # COS
-          resp=create_token_www_body({
-            'grant_type'          => 'urn:ibm:params:oauth:grant-type:apikey',
-            'response_type'       => 'delegated_refresh_token',
-            'apikey'              => @params[:api_key],
-            'receiver_client_ids' => 'aspera_ats'
+          resp=create_token(www_body_params: {
+            grant_type:          'urn:ibm:params:oauth:grant-type:apikey',
+            response_type:       'delegated_refresh_token',
+            apikey:              @params[:api_key],
+            receiver_client_ids: 'aspera_ats'
           })
         when :header_userpass
           # used in Faspex apiv4 and shares2
-          resp=create_token_advanced({
-            :auth        => {
-            :type          => :basic,
-            :username      => @params[:user_name],
-            :password      => @params[:user_pass]},
-            :json_params => p_client_id_and_scope.merge({:grant_type => 'password'}), #:www_body_params also works
-          })
+          resp=create_token(
+          json_params: p_client_id_and_scope.merge({grant_type: 'password'}), #:www_body_params also works
+          auth:        {
+            type:     :basic,
+            username: @params[:user_name],
+            password: @params[:user_pass]}
+          )
         when :body_userpass
           # legacy, not used
-          resp=create_token_www_body(p_client_id_and_scope.merge({
-            :grant_type => 'password',
-            :username   => @params[:user_name],
-            :password   => @params[:user_pass]
+          resp=create_token(www_body_params: p_client_id_and_scope.merge({
+            grant_type: 'password',
+            username:   @params[:user_name],
+            password:   @params[:user_pass]
           }))
         when :body_data
           # used in Faspex apiv5
-          resp=create_token_advanced({
-            :auth        => {:type => :none},
-            :json_params => @params[:userpass_body],
+          resp=create_token({
+            auth:        {type: :none},
+            json_params: @params[:userpass_body],
           })
         else
           raise "auth grant type unknown: #{@params[:grant]}"
@@ -311,9 +327,9 @@ module Aspera
         # TODO: test return code ?
         json_data=resp[:http].body
         token_data=JSON.parse(json_data)
-        self.class.persist_mgr.put(token_ids,json_data)
+        self.class.persist_mgr.put(token_id,json_data)
       end # if ! in_cache
-
+      raise "API error: No such field in answer: #{@params[:token_field]}" unless token_data.has_key?(@params[:token_field])
       # ok we shall have a token here
       return 'Bearer '+token_data[@params[:token_field]]
     end

data/lib/aspera/persistency_action_once.rb

@@ -6,7 +6,7 @@ module Aspera
   class PersistencyActionOnce
     # @param :manager  Mandatory Database
     # @param :data     Mandatory object to persist, must be same object from begin to end (assume array by default)
-    # @param :ids      Mandatory identifiers
+    # @param :id      Mandatory identifiers
     # @param :delete   Optional  delete persistency condition
     # @param :parse    Optional  parse method (default to JSON)
     # @param :format   Optional  dump method (default to JSON)
@@ -16,27 +16,31 @@ module Aspera
       raise "options shall be Hash" unless options.is_a?(Hash)
       raise "mandatory :manager" if options[:manager].nil?
       raise "mandatory :data" if options[:data].nil?
-      raise "mandatory :ids (Array)" unless options[:ids].is_a?(Array)
-      raise "mandatory 1 element in :ids" unless options[:ids].length >= 1
+      raise "mandatory :id (String)" unless options[:id].is_a?(String)
+      raise "mandatory 1 element in :id" unless options[:id].length >= 1
       @manager=options[:manager]
       @persisted_object=options[:data]
-      @object_ids=options[:ids]
+      @object_id=options[:id]
       # by default , at save time, file is deleted if data is nil
       @delete_condition=options[:delete] || lambda{|d|d.empty?}
       @persist_format=options[:format] || lambda {|h| JSON.generate(h)}
       persist_parse=options[:parse] || lambda {|t| JSON.parse(t)}
       persist_merge=options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
-      value=@manager.get(@object_ids)
+      value=@manager.get(@object_id)
       persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
     end
 
     def save
       if @delete_condition.call(@persisted_object)
-        @manager.delete(@object_ids)
+        @manager.delete(@object_id)
       else
-        @manager.put(@object_ids,@persist_format.call(@persisted_object))
+        @manager.put(@object_id,@persist_format.call(@persisted_object))
       end
     end
 
+    def data
+      return @persisted_object
+    end
+
   end
 end

data/lib/aspera/persistency_folder.rb

@@ -6,11 +6,8 @@ require 'aspera/log'
 module Aspera
   # Persist data on file system
   class PersistencyFolder
-    WINDOWS_PROTECTED_CHAR=%r{[/:"<>\\\*\?]}
-    PROTECTED_CHAR_REPLACE='_'
-    ID_SEPARATOR='_'
     FILE_SUFFIX='.txt'
-    private_constant :PROTECTED_CHAR_REPLACE,:FILE_SUFFIX,:WINDOWS_PROTECTED_CHAR,:ID_SEPARATOR
+    private_constant :FILE_SUFFIX
     def initialize(folder)
       @cache={}
       set_folder(folder)
@@ -23,7 +20,6 @@ module Aspera
 
     # @return String or nil string on existing persist, else nil
     def get(object_id)
-      object_id=marshalled_id(object_id)
       Log.log.debug("persistency get: #{object_id}")
       if @cache.has_key?(object_id)
         Log.log.debug("got from memory cache")
@@ -39,18 +35,16 @@ module Aspera
     end
 
     def put(object_id,value)
-      raise "only String supported" unless value.is_a?(String)
-      object_id=marshalled_id(object_id)
+      raise "value: only String supported" unless value.is_a?(String)
       persist_filepath=id_to_filepath(object_id)
-      Log.log.debug("saving: #{persist_filepath}")
+      Log.log.debug("persistency saving: #{persist_filepath}")
       File.write(persist_filepath,value)
       @cache[object_id]=value
     end
 
     def delete(object_id)
-      object_id=marshalled_id(object_id)
       persist_filepath=id_to_filepath(object_id)
-      Log.log.debug("empty data, deleting: #{persist_filepath}")
+      Log.log.debug("persistency deleting: #{persist_filepath}")
       File.delete(persist_filepath) if File.exist?(persist_filepath)
       @cache.delete(object_id)
     end
@@ -63,7 +57,7 @@ module Aspera
       end
       garbage_files.each do |filepath|
         File.delete(filepath)
-        Log.log.debug("Deleted expired: #{filepath}")
+        Log.log.debug("persistency deleted expired: #{filepath}")
       end
       return garbage_files
     end
@@ -72,25 +66,11 @@ module Aspera
 
     # @param object_id String or Array
     def id_to_filepath(object_id)
+      raise "object_id: only String supported" unless object_id.is_a?(String)
       FileUtils.mkdir_p(@folder)
       return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
       #.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
     end
 
-    def marshalled_id(object_id)
-      if object_id.is_a?(Array)
-        # special case, url in second position: TODO: check any position
-        if object_id[1].is_a?(String) and object_id[1] =~ URI::ABS_URI
-          object_id=object_id.clone
-          object_id[1]=URI.parse(object_id[1]).host
-        end
-        object_id=object_id.join(ID_SEPARATOR)
-      end
-      raise "id must be a String" unless object_id.is_a?(String)
-      return object_id.
-      gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
-      gsub('.',PROTECTED_CHAR_REPLACE).  # keep dot for extension only (nicer)
-      downcase
-    end
   end # PersistencyFolder
 end # Aspera