aspera-cli 4.19.0 → 4.20.0

data/lib/aspera/oauth/base.rb

@@ -3,64 +3,66 @@
 require 'aspera/oauth/factory'
 require 'aspera/log'
 require 'aspera/assert'
-require 'aspera/id_generator'
 require 'date'
 
 module Aspera
   module OAuth
-    # Implement OAuth 2 for the REST client and generate a bearer token
-    # bearer tokens are cached in memory and in a file cache for later re-use
+    # OAuth 2 client for the REST client
+    # Generate bearer token
+    # Bearer tokens are cached in memory and in a file cache for later re-use
     # https://tools.ietf.org/html/rfc6749
     class Base
-      # scope can be modified after creation
-      attr_writer :scope
-
-      # [M]=mandatory [D]=has default value [O]=Optional/nil
-      # @param base_url            [M] URL of authentication API
-      # @param auth                [O] basic auth parameters
-      # @param client_id           [O]
-      # @param client_secret       [O]
-      # @param scope               [O]
-      # @param path_token          [D] API end point to create a token
-      # @param token_field         [D] field in result that contains the token
+      # @param **             Parameters for REST
+      # @param client_id      [String, nil]
+      # @param client_secret  [String, nil]
+      # @param scope          [String, nil]
+      # @param use_query      [bool]        Provide parameters in query instead of body
+      # @param path_token     [String]      API end point to create a token
+      # @param token_field    [String]      Field in result that contains the token
+      # @param cache_ids      [Array, nil]  List of unique identifiers for cache id generation
       def initialize(
-        base_url:,
-        auth: nil,
         client_id: nil,
         client_secret: nil,
         scope: nil,
         use_query: false,
-        path_token:  'token',       # default endpoint for /token to generate token
-        token_field: 'access_token' # field with token in result of call to path_token
+        path_token:  'token',
+        token_field: 'access_token',
+        cache_ids: nil,
+        **rest_params
       )
-        Aspera.assert_type(base_url, String)
         Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', exception_class: InternalError)
-        @base_url = base_url
+        # this is the OAuth API
+        @api = Rest.new(**rest_params)
         @path_token = path_token
         @token_field = token_field
         @client_id = client_id
         @client_secret = client_secret
-        @scope = scope
         @use_query = use_query
-        @identifiers = []
-        @identifiers.push(auth[:username]) if auth.is_a?(Hash) && auth.key?(:username)
-        # this is the OAuth API
-        @api = Rest.new(
-          base_url:     @base_url,
-          redirect_max: 2,
-          auth:         auth)
+        @base_cache_ids = cache_ids.clone
+        @base_cache_ids = [] if @base_cache_ids.nil?
+        Aspera.assert_type(@base_cache_ids, Array)
+        if @api.auth_params.key?(:username)
+          cache_ids.push(@api.auth_params[:username])
+        end
+        @base_cache_ids.freeze
+        self.scope = scope
+      end
+
+      # Scope can be modified after creation, then update identifier for cache
+      def scope=(scope)
+        @scope = scope
+        # generate token unique identifier for persistency (memory/disk cache)
+        @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
       end
 
       # helper method to create token as per RFC
       def create_token_call(creation_params)
         Log.log.debug{'Generating a new token'.bg_green}
-        payload = {
-          body:      creation_params,
-          body_type: :www
-        }
+        payload = { body_type: :www }
         if @use_query
           payload[:query] = creation_params
-          payload[:body] = {}
+        else
+          payload[:body] = creation_params
         end
         return @api.call(
           operation: 'POST',
@@ -85,16 +87,8 @@ module Aspera
       # @param cache set to false to disable cache
       # @param refresh set to true to force refresh or re-generation (if previous failed)
       def token(cache: true, refresh: false)
-        # generate token unique identifier for persistency (memory/disk cache)
-        token_id = IdGenerator.from_list(Factory.id(
-          @base_url,
-          Factory.class_to_id(self.class),
-          @identifiers,
-          @scope
-        ))
-
         # get token_data from cache (or nil), token_data is what is returned by /token
-        token_data = Factory.instance.persist_mgr.get(token_id) if cache
+        token_data = Factory.instance.persist_mgr.get(@token_cache_id) if cache
         token_data = JSON.parse(token_data) unless token_data.nil?
         # Optional optimization: check if node token is expired based on decoded content then force refresh if close enough
         # might help in case the transfer agent cannot refresh himself
@@ -120,7 +114,7 @@ module Aspera
             refresh_token = token_data['refresh_token']
           end
           # delete cache
-          Factory.instance.persist_mgr.delete(token_id)
+          Factory.instance.persist_mgr.delete(@token_cache_id)
           token_data = nil
           # lets try the existing refresh token
           if !refresh_token.nil?
@@ -132,7 +126,7 @@ module Aspera
               # save only if success
               json_data = resp[:http].body
               token_data = JSON.parse(json_data)
-              Factory.instance.persist_mgr.put(token_id, json_data)
+              Factory.instance.persist_mgr.put(@token_cache_id, json_data)
             else
               Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
             end
@@ -144,7 +138,7 @@ module Aspera
           resp = create_token
           json_data = resp[:http].body
           token_data = JSON.parse(json_data)
-          Factory.instance.persist_mgr.put(token_id, json_data)
+          Factory.instance.persist_mgr.put(@token_cache_id, json_data)
         end
         Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"}
         # ok we shall have a token here

data/lib/aspera/oauth/factory.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require 'singleton'
+require 'aspera/id_generator'
 require 'aspera/assert'
+require 'singleton'
 require 'base64'
 module Aspera
   module OAuth
@@ -29,11 +30,17 @@ module Aspera
           return token[BEARER_PREFIX.length..-1]
         end
 
-        def id(*params)
-          return [PERSIST_CATEGORY_TOKEN, *params].flatten
+        # @return a cache identifier
+        def cache_id(url, creator_class, *params)
+          return IdGenerator.from_list([
+            PERSIST_CATEGORY_TOKEN,
+            url,
+            Factory.class_to_id(creator_class),
+            *params
+          ].flatten)
         end
 
-        # snake version of class name is the identifier
+        # @return snake version of class name
         def class_to_id(creator_class)
           return creator_class.name.split('::').last.capital_to_snake.to_sym
         end

data/lib/aspera/oauth/generic.rb

@@ -13,17 +13,13 @@ module Aspera
         receiver_client_ids: nil,
         **base_params
       )
-        super(**base_params)
+        super(**base_params, cache_ids: [grant_type&.split(':')&.last, apikey, response_type])
         @create_params = {
           grant_type: grant_type
         }
-        @create_params[:response_type] = response_type if response_type
-        @create_params[:apikey] = apikey if apikey
-        @create_params[:receiver_client_ids] = receiver_client_ids if receiver_client_ids
-        @identifiers.push(
-          @create_params[:grant_type]&.split(':')&.last,
-          @create_params[:apikey],
-          @create_params[:response_type])
+        @create_params[:response_type] = response_type unless response_type.nil?
+        @create_params[:apikey] = apikey unless apikey.nil?
+        @create_params[:receiver_client_ids] = receiver_client_ids unless receiver_client_ids.nil?
       end
 
       def create_token

data/lib/aspera/oauth/jwt.rb

@@ -13,6 +13,7 @@ module Aspera
     # https://tools.ietf.org/html/rfc7523
     # https://tools.ietf.org/html/rfc7519
     class Jwt < Base
+      GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
       # @param private_key_obj private key object
       # @param payload payload to be included in the JWT
       # @param headers headers to be included in the JWT
@@ -25,11 +26,10 @@ module Aspera
         Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
         Aspera.assert_type(payload, Hash){'payload'}
         Aspera.assert_type(headers, Hash){'headers'}
-        super(**base_params)
+        super(**base_params, cache_ids: [payload[:sub]])
         @private_key_obj = private_key_obj
         @additional_payload = payload
         @headers = headers
-        @identifiers.push(@additional_payload[:sub])
       end
 
       def create_token
@@ -46,7 +46,7 @@ module Aspera
         Log.log.debug{"private=[#{@private_key_obj}]"}
         assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
         Log.log.debug{"assertion=[#{assertion}]"}
-        return create_token_call(optional_scope_client_id.merge(grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: assertion))
+        return create_token_call(optional_scope_client_id.merge(grant_type: GRANT_TYPE, assertion: assertion))
       end
     end
     Factory.instance.register_token_creator(Jwt)

data/lib/aspera/oauth/url_json.rb

@@ -13,10 +13,9 @@ module Aspera
         json:,
         **generic_params
       )
-        super(**generic_params)
+        super(**generic_params, cache_ids: [json[:url_token]])
         @body = json
         @query = url
-        @identifiers.push(@body[:url_token])
       end
 
       def create_token

data/lib/aspera/oauth/web.rb

@@ -8,6 +8,9 @@ module Aspera
   module OAuth
     # Authentication using Web browser
     class Web < Base
+      class << self
+        attr_accessor :additionnal_info
+      end
       # @param redirect_uri    url to receive the code after auth (to be exchanged for token)
       # @param path_authorize  path to login page on web app
       def initialize(
@@ -28,12 +31,12 @@ module Aspera
         # generate secure state to check later
         random_state = SecureRandom.uuid
         login_page_url = Rest.build_uri(
-          "#{@base_url}/#{@path_authorize}",
+          "#{@api.base_url}/#{@path_authorize}",
           optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state))
         # here, we need a human to authorize on a web page
         Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
         # start a web server to receive request code
-        web_server = WebAuth.new(@redirect_uri)
+        web_server = WebAuth.new(@redirect_uri, self.class.additionnal_info)
         # start browser on login page
         Environment.instance.open_uri(login_page_url)
         # wait for code in request

data/lib/aspera/persistency_action_once.rb

@@ -7,6 +7,13 @@ require 'aspera/assert'
 module Aspera
   # Persist data on file system
   class PersistencyActionOnce
+    DELETE_DEFAULT = lambda{|d|d.empty?}
+    PARSE_DEFAULT = lambda {|t| JSON.parse(t)}
+    FORMAT_DEFAULT = lambda {|h| JSON.generate(h)}
+    MERGE_DEFAULT = lambda {|current, file| current.concat(file).uniq rescue current}
+    MANAGER_METHODS = %i[get put delete]
+    private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
+
     # @param :manager  Mandatory Database
     # @param :data     Mandatory object to persist, must be same object from begin to end (assume array by default)
     # @param :id       Mandatory identifiers
@@ -14,21 +21,22 @@ module Aspera
     # @param :parse    Optional  parse method (default to JSON)
     # @param :format   Optional  dump method (default to JSON)
     # @param :merge    Optional  merge data from file to current data
-    def initialize(manager:, data:, id:, delete: nil, parse: nil, format: nil, merge: nil)
-      Aspera.assert(!manager.nil?)
+    def initialize(manager:, data:, id:, delete: DELETE_DEFAULT, parse: PARSE_DEFAULT, format: FORMAT_DEFAULT, merge: MERGE_DEFAULT)
+      Aspera.assert(MANAGER_METHODS.all?{|i|manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
       Aspera.assert(!data.nil?)
       Aspera.assert_type(id, String)
       Aspera.assert(!id.empty?)
+      Aspera.assert_type(delete, Proc)
+      Aspera.assert_type(parse, Proc)
+      Aspera.assert_type(format, Proc)
+      Aspera.assert_type(merge, Proc)
       @manager = manager
       @persisted_object = data
       @object_id = id
-      # by default , at save time, file is deleted if data is nil
-      @delete_condition = delete || lambda{|d|d.empty?}
-      @persist_format = format || lambda {|h| JSON.generate(h)}
-      persist_parse = parse || lambda {|t| JSON.parse(t)}
-      persist_merge = merge || lambda {|current, file| current.concat(file).uniq rescue current}
+      @delete_condition = delete
+      @persist_format = format
       value = @manager.get(@object_id)
-      persist_merge.call(@persisted_object, persist_parse.call(value)) unless value.nil?
+      merge.call(@persisted_object, parse.call(value)) unless value.nil?
     end
 
     def save

data/lib/aspera/preview/utils.rb

@@ -32,7 +32,7 @@ module Aspera
           tools_to_check.delete(:unoconv) if skip_types.include?(:office)
           # Check for binaries
           tools_to_check.each do |command_sym|
-            external_command(command_sym, ['-h'], check_code: false)
+            external_command(command_sym, ['-h'])
           rescue Errno::ENOENT => e
             raise "missing #{command_sym} binary: #{e}"
           rescue
@@ -43,19 +43,9 @@ module Aspera
         # execute external command
         # one could use "system", but we would need to redirect stdout/err
         # @return true if su
-        def external_command(command_sym, command_args, check_code: true)
+        def external_command(command_sym, command_args)
           Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
-          # build command line, and quote special characters
-          command_line = command_args.clone.unshift(command_sym).map{|i| shell_quote(i.to_s)}.join(' ')
-          Log.log.debug{"cmd=#{command_line}".blue}
-          stdout, stderr, status = Open3.capture3(command_line)
-          if check_code && !status.success?
-            Log.log.error{"status: #{status}"}
-            Log.log.error{"stdout: #{stdout}"}
-            Log.log.error{"stderr: #{stderr}"}
-            raise "#{command_sym} error #{status}"
-          end
-          return {status: status, stdout: stdout}
+          return Environment.secure_capture(command_sym.to_s, *command_args)
         end
 
         def ffmpeg(a)
@@ -73,12 +63,11 @@ module Aspera
 
         # @return Float in seconds
         def video_get_duration(input_file)
-          result = external_command(:ffprobe, [
+          return external_command(:ffprobe, [
             '-loglevel', 'error',
             '-show_entries', 'format=duration',
             '-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
-            input_file])
-          return result[:stdout].to_f
+            input_file]).to_f
         end
 
         def ffmpeg_fmt(temp_folder)