aspera-cli 4.19.0 → 4.21.1

data/lib/aspera/oauth/factory.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require 'singleton'
+require 'aspera/id_generator'
 require 'aspera/assert'
+require 'singleton'
 require 'base64'
 module Aspera
   module OAuth
@@ -12,6 +13,7 @@ module Aspera
       PERSIST_CATEGORY_TOKEN = 'token'
       # prefix for bearer token when in header
       BEARER_PREFIX = 'Bearer '
+      TOKEN_FIELD = 'access_token'
 
       private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
 
@@ -29,11 +31,17 @@ module Aspera
           return token[BEARER_PREFIX.length..-1]
         end
 
-        def id(*params)
-          return [PERSIST_CATEGORY_TOKEN, *params].flatten
+        # @return a cache identifier
+        def cache_id(url, creator_class, *params)
+          return IdGenerator.from_list([
+            PERSIST_CATEGORY_TOKEN,
+            url,
+            Factory.class_to_id(creator_class),
+            *params
+          ].flatten)
         end
 
-        # snake version of class name is the identifier
+        # @return snake version of class name
         def class_to_id(creator_class)
           return creator_class.name.split('::').last.capital_to_snake.to_sym
         end
@@ -80,7 +88,45 @@ module Aspera
 
       # delete all existing tokens
       def flush_tokens
-        persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
+        persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN)
+      end
+
+      def persisted_tokens
+        data = persist_mgr.current_items(PERSIST_CATEGORY_TOKEN)
+        data.each.map do |k, v|
+          info = {id: k}
+          info.merge!(JSON.parse(v)) rescue nil
+          d = decode_token(info.delete(TOKEN_FIELD))
+          info.merge(d) if d
+          info
+        end
+      end
+
+      # get token information from cache
+      # @param id [String] identifier of token
+      # @return [Hash] token internal information , including Date object for `expiration_date`
+      def get_token_info(id)
+        token_raw_string = persist_mgr.get(id)
+        return nil if token_raw_string.nil?
+        token_data = JSON.parse(token_raw_string)
+        Aspera.assert_type(token_data, Hash)
+        decoded_token = decode_token(token_data[TOKEN_FIELD])
+        info = { data: token_data }
+        Log.log.debug{Log.dump('decoded_token', decoded_token)}
+        if decoded_token.is_a?(Hash)
+          info[:decoded] = decoded_token
+          # TODO: move date decoding to token decoder ?
+          expiration_date =
+            if    decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
+            elsif decoded_token['exp'].is_a?(Integer)       then Time.at(decoded_token['exp'])
+            end
+          unless expiration_date.nil?
+            info[:expiration] = expiration_date
+            info[:ttl_sec] = expiration_date - Time.now
+            info[:expired] = info[:ttl_sec] < @parameters[:token_expiration_guard_sec]
+          end
+        end
+        return info
       end
 
       # register a bearer token decoder, mainly to inspect expiry date
@@ -118,6 +164,6 @@ module Aspera
       end
     end
     # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
-    Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not aoc token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
+    Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
   end
 end

data/lib/aspera/oauth/generic.rb

@@ -13,17 +13,13 @@ module Aspera
         receiver_client_ids: nil,
         **base_params
       )
-        super(**base_params)
+        super(**base_params, cache_ids: [grant_type&.split(':')&.last, apikey, response_type])
         @create_params = {
           grant_type: grant_type
         }
-        @create_params[:response_type] = response_type if response_type
-        @create_params[:apikey] = apikey if apikey
-        @create_params[:receiver_client_ids] = receiver_client_ids if receiver_client_ids
-        @identifiers.push(
-          @create_params[:grant_type]&.split(':')&.last,
-          @create_params[:apikey],
-          @create_params[:response_type])
+        @create_params[:response_type] = response_type unless response_type.nil?
+        @create_params[:apikey] = apikey unless apikey.nil?
+        @create_params[:receiver_client_ids] = receiver_client_ids unless receiver_client_ids.nil?
       end
 
       def create_token

data/lib/aspera/oauth/jwt.rb

@@ -13,6 +13,7 @@ module Aspera
     # https://tools.ietf.org/html/rfc7523
     # https://tools.ietf.org/html/rfc7519
     class Jwt < Base
+      GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
       # @param private_key_obj private key object
       # @param payload payload to be included in the JWT
       # @param headers headers to be included in the JWT
@@ -20,16 +21,18 @@ module Aspera
         private_key_obj:,
         payload:,
         headers: {},
+        cache_ids: [],
         **base_params
       )
         Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
         Aspera.assert_type(payload, Hash){'payload'}
         Aspera.assert_type(headers, Hash){'headers'}
-        super(**base_params)
+        Aspera.assert_type(cache_ids, Array){'cache ids'}
+        new_cache_ids = cache_ids.clone.push(payload[:sub])
+        super(**base_params, cache_ids: new_cache_ids)
         @private_key_obj = private_key_obj
         @additional_payload = payload
         @headers = headers
-        @identifiers.push(@additional_payload[:sub])
       end
 
       def create_token
@@ -46,7 +49,7 @@ module Aspera
         Log.log.debug{"private=[#{@private_key_obj}]"}
         assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
         Log.log.debug{"assertion=[#{assertion}]"}
-        return create_token_call(optional_scope_client_id.merge(grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: assertion))
+        return create_token_call(optional_scope_client_id.merge(grant_type: GRANT_TYPE, assertion: assertion))
       end
     end
     Factory.instance.register_token_creator(Jwt)

data/lib/aspera/oauth/url_json.rb

@@ -13,10 +13,9 @@ module Aspera
         json:,
         **generic_params
       )
-        super(**generic_params)
+        super(**generic_params, cache_ids: [json[:url_token]])
         @body = json
         @query = url
-        @identifiers.push(@body[:url_token])
       end
 
       def create_token

data/lib/aspera/oauth/web.rb

@@ -8,6 +8,9 @@ module Aspera
   module OAuth
     # Authentication using Web browser
     class Web < Base
+      class << self
+        attr_accessor :additionnal_info
+      end
       # @param redirect_uri    url to receive the code after auth (to be exchanged for token)
       # @param path_authorize  path to login page on web app
       def initialize(
@@ -28,12 +31,12 @@ module Aspera
         # generate secure state to check later
         random_state = SecureRandom.uuid
         login_page_url = Rest.build_uri(
-          "#{@base_url}/#{@path_authorize}",
+          "#{@api.base_url}/#{@path_authorize}",
           optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state))
         # here, we need a human to authorize on a web page
         Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
         # start a web server to receive request code
-        web_server = WebAuth.new(@redirect_uri)
+        web_server = WebAuth.new(@redirect_uri, self.class.additionnal_info)
         # start browser on login page
         Environment.instance.open_uri(login_page_url)
         # wait for code in request

data/lib/aspera/persistency_action_once.rb

@@ -7,6 +7,13 @@ require 'aspera/assert'
 module Aspera
   # Persist data on file system
   class PersistencyActionOnce
+    DELETE_DEFAULT = lambda(&:empty?)
+    PARSE_DEFAULT = lambda {|t| JSON.parse(t)}
+    FORMAT_DEFAULT = lambda {|h| JSON.generate(h)}
+    MERGE_DEFAULT = lambda {|current, file| current.concat(file).uniq rescue current}
+    MANAGER_METHODS = %i[get put delete]
+    private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
+
     # @param :manager  Mandatory Database
     # @param :data     Mandatory object to persist, must be same object from begin to end (assume array by default)
     # @param :id       Mandatory identifiers
@@ -14,21 +21,22 @@ module Aspera
     # @param :parse    Optional  parse method (default to JSON)
     # @param :format   Optional  dump method (default to JSON)
     # @param :merge    Optional  merge data from file to current data
-    def initialize(manager:, data:, id:, delete: nil, parse: nil, format: nil, merge: nil)
-      Aspera.assert(!manager.nil?)
+    def initialize(manager:, data:, id:, delete: DELETE_DEFAULT, parse: PARSE_DEFAULT, format: FORMAT_DEFAULT, merge: MERGE_DEFAULT)
+      Aspera.assert(MANAGER_METHODS.all?{|i|manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
       Aspera.assert(!data.nil?)
       Aspera.assert_type(id, String)
       Aspera.assert(!id.empty?)
+      Aspera.assert_type(delete, Proc)
+      Aspera.assert_type(parse, Proc)
+      Aspera.assert_type(format, Proc)
+      Aspera.assert_type(merge, Proc)
       @manager = manager
       @persisted_object = data
       @object_id = id
-      # by default , at save time, file is deleted if data is nil
-      @delete_condition = delete || lambda{|d|d.empty?}
-      @persist_format = format || lambda {|h| JSON.generate(h)}
-      persist_parse = parse || lambda {|t| JSON.parse(t)}
-      persist_merge = merge || lambda {|current, file| current.concat(file).uniq rescue current}
+      @delete_condition = delete
+      @persist_format = format
       value = @manager.get(@object_id)
-      persist_merge.call(@persisted_object, persist_parse.call(value)) unless value.nil?
+      merge.call(@persisted_object, parse.call(value)) unless value.nil?
     end
 
     def save

data/lib/aspera/persistency_folder.rb

@@ -18,7 +18,8 @@ module Aspera
       Log.log.debug{"persistency folder: #{@folder}"}
     end
 
-    # @return String or nil string on existing persist, else nil
+    # Get value of persisted item
+    # @return [String,nil] Value of persisted id
     def get(object_id)
       Log.log.debug{"persistency get: #{object_id}"}
       if @cache.key?(object_id)
@@ -34,6 +35,10 @@ module Aspera
       return @cache[object_id]
     end
 
+    # Set value of persisted item
+    # @param object_id [String] Identifier of persisted item
+    # @param value [String] Value of persisted item
+    # @return [nil]
     def put(object_id, value)
       Aspera.assert_type(value, String)
       persist_filepath = id_to_filepath(object_id)
@@ -42,8 +47,11 @@ module Aspera
       File.write(persist_filepath, value)
       Environment.restrict_file_access(persist_filepath)
       @cache[object_id] = value
+      nil
     end
 
+    # Delete persisted item
+    # @param object_id [String] Identifier of persisted item
     def delete(object_id)
       persist_filepath = id_to_filepath(object_id)
       Log.log.debug{"persistency deleting: #{persist_filepath}"}
@@ -51,8 +59,9 @@ module Aspera
       @cache.delete(object_id)
     end
 
+    # Delete persisted items
     def garbage_collect(persist_category, max_age_seconds=nil)
-      garbage_files = Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
+      garbage_files = current_files(persist_category)
       if !max_age_seconds.nil?
         current_time = Time.now
         garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
@@ -61,9 +70,18 @@ module Aspera
         File.delete(filepath)
         Log.log.debug{"persistency deleted expired: #{filepath}"}
       end
+      @cache.clear
       return garbage_files
     end
 
+    def current_files(persist_category)
+      Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
+    end
+
+    def current_items(persist_category)
+      current_files(persist_category).each_with_object({}) {|i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
+    end
+
     private
 
     # @param object_id String or Array

data/lib/aspera/preview/generator.rb

@@ -66,7 +66,7 @@ module Aspera
           result_size = File.size(@destination_file_path)
           Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"} if result_size > @options.max_size
         rescue StandardError => e
-          Log.log.error{"Ignoring: #{e.message}"}
+          Log.log.error{"Ignoring: #{e.class} #{e.message}"}
           Log.log.debug(e.backtrace.join("\n").red)
           FileUtils.cp(File.expand_path(@preview_format_sym.eql?(:mp4) ? 'video_error.png' : 'image_error.png', File.dirname(__FILE__)), @destination_file_path)
         ensure

data/lib/aspera/preview/utils.rb

@@ -32,7 +32,7 @@ module Aspera
           tools_to_check.delete(:unoconv) if skip_types.include?(:office)
           # Check for binaries
           tools_to_check.each do |command_sym|
-            external_command(command_sym, ['-h'], check_code: false)
+            external_command(command_sym, ['-h'])
           rescue Errno::ENOENT => e
             raise "missing #{command_sym} binary: #{e}"
           rescue
@@ -42,20 +42,15 @@ module Aspera
 
         # execute external command
         # one could use "system", but we would need to redirect stdout/err
-        # @return true if su
-        def external_command(command_sym, command_args, check_code: true)
+        # @return nil
+        def external_command(command_sym, command_args)
           Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
-          # build command line, and quote special characters
-          command_line = command_args.clone.unshift(command_sym).map{|i| shell_quote(i.to_s)}.join(' ')
-          Log.log.debug{"cmd=#{command_line}".blue}
-          stdout, stderr, status = Open3.capture3(command_line)
-          if check_code && !status.success?
-            Log.log.error{"status: #{status}"}
-            Log.log.error{"stdout: #{stdout}"}
-            Log.log.error{"stderr: #{stderr}"}
-            raise "#{command_sym} error #{status}"
-          end
-          return {status: status, stdout: stdout}
+          Environment.secure_execute(exec: command_sym.to_s, args: command_args.map(&:to_s))
+        end
+
+        def external_capture(command_sym, command_args)
+          Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
+          return Environment.secure_capture(exec: command_sym.to_s, args: command_args.map(&:to_s))
         end
 
         def ffmpeg(a)
@@ -73,12 +68,11 @@ module Aspera
 
         # @return Float in seconds
         def video_get_duration(input_file)
-          result = external_command(:ffprobe, [
+          return external_capture(:ffprobe, [
             '-loglevel', 'error',
             '-show_entries', 'format=duration',
             '-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
-            input_file])
-          return result[:stdout].to_f
+            input_file]).to_f
         end
 
         def ffmpeg_fmt(temp_folder)

data/lib/aspera/products/alpha.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'aspera/environment'
+
+module Aspera
+  module Products
+    # Aspera Desktop Alpha Client
+    class Alpha
+      APP_NAME = 'IBM Aspera for Desktop'
+      APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
+      class << self
+        # standard folder locations
+        def locations
+          case Aspera::Environment.os
+          when Aspera::Environment::OS_MACOS then [{
+            app_root: File.join('', 'Applications', 'IBM Aspera.app'),
+            log_root: File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER),
+            sub_bin:  File.join('Contents', 'Resources', 'transferd', 'bin')
+          }]
+          else []
+          end.map { |i| i.merge({ expected: APP_NAME }) }
+        end
+
+        def log_file
+          File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
+        end
+      end
+    end
+  end
+end

data/lib/aspera/products/connect.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'aspera/environment'
+
+module Aspera
+  module Products
+    class Connect
+      APP_NAME = 'IBM Aspera Connect'
+      class << self
+        # standard folder locations
+        def locations
+          case Aspera::Environment.os
+          when Aspera::Environment::OS_WINDOWS then [{
+            app_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Programs', 'Aspera', 'Aspera Connect'),
+            log_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect', 'var', 'log'),
+            run_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect')
+          }]
+          when Aspera::Environment::OS_MACOS then [{
+            app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
+            log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
+            run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
+            sub_bin:  File.join('Contents', 'Resources')
+          }, {
+            app_root: File.join('', 'Applications', 'Aspera Connect.app'),
+            log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
+            run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
+            sub_bin:  File.join('Contents', 'Resources')
+          }, {
+            app_root: File.join(Dir.home, 'Applications', 'IBM Aspera Connect.app'),
+            log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
+            run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
+            sub_bin:  File.join('Contents', 'Resources')
+          }, {
+            app_root: File.join('', 'Applications', 'IBM Aspera Connect.app'),
+            log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
+            run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
+            sub_bin:  File.join('Contents', 'Resources')
+          }]
+          else [{ # other: Linux and Unix family
+            app_root: File.join(Dir.home, '.aspera', 'connect'),
+            run_root: File.join(Dir.home, '.aspera', 'connect')
+          }]
+          end.map { |i| i.merge({ expected: APP_NAME }) }
+        end
+      end
+    end
+  end
+end

data/lib/aspera/products/other.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# cspell:ignore LOCALAPPDATA
+require 'aspera/environment'
+
+module Aspera
+  # Location of Aspera products, for which an Agent is not proposed
+  module Products
+    # other Aspera products with ascp
+    class Other
+      CLI_V3 = 'Aspera CLI (deprecated)'
+      DRIVE = 'Aspera Drive (deprecated)'
+      HSTS = 'IBM Aspera High-Speed Transfer Server'
+
+      private_constant :CLI_V3, :DRIVE, :HSTS
+      # product information manifest: XML (part of aspera product)
+      INFO_META_FILE = 'product-info.mf'
+
+      # :expected  M app name is taken from the manifest if present, else defaults to this value
+      # :app_root  M main folder for the application
+      # :log_root  O location of log files (Linux uses syslog)
+      # :run_root  O only for Connect Client, location of http port file
+      # :sub_bin   O subfolder with executables, default : bin
+      LOCATION_ON_THIS_OS = case Aspera::Environment.os
+      when Aspera::Environment::OS_WINDOWS then [{
+        expected: CLI_V3,
+        app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
+        log_root: File.join('C:', 'Program Files', 'Aspera', 'cli', 'var', 'log')
+      }, {
+        expected: HSTS,
+        app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
+        log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
+      }]
+      when Aspera::Environment::OS_MACOS then [{
+        expected: CLI_V3,
+        app_root: File.join(Dir.home, 'Applications', 'Aspera CLI'),
+        log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
+      }, {
+        expected: HSTS,
+        app_root: File.join('', 'Library', 'Aspera'),
+        log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
+      }, {
+        expected: DRIVE,
+        app_root: File.join('', 'Applications', 'Aspera Drive.app'),
+        log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Drive'),
+        sub_bin:  File.join('Contents', 'Resources')
+      }]
+      else [{ # other: Linux and Unix family
+        expected: CLI_V3,
+        app_root: File.join(Dir.home, '.aspera', 'cli')
+      }, {
+        expected: HSTS,
+        app_root: File.join('', 'opt', 'aspera')
+      }]
+      end
+      class << self
+        def find(scan_locations)
+          scan_locations.select do |item|
+            # skip if not main folder
+            Log.log.trace1{"Checking #{item[:app_root]}"}
+            next false unless Dir.exist?(item[:app_root])
+            Log.log.debug{"Found #{item[:expected]}"}
+            sub_bin = item[:sub_bin] || 'bin'
+            item[:ascp_path] = File.join(item[:app_root], sub_bin, Environment.exe_file('ascp'))
+            # skip if no ascp
+            next false unless File.exist?(item[:ascp_path])
+            # read info from product info file if present
+            product_info_file = "#{item[:app_root]}/#{INFO_META_FILE}"
+            if File.exist?(product_info_file)
+              res_s = XmlSimple.xml_in(File.read(product_info_file), {'ForceArray' => false})
+              item[:name] = res_s['name']
+              item[:version] = res_s['version']
+            else
+              item[:name] = item[:expected]
+            end
+            true # select this version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aspera/products/transferd.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Aspera
+  module Products
+    class Transferd
+      APP_NAME = 'IBM Aspera Transfer Daemon'
+      class << self
+        # standard folder locations
+        def locations
+          [{
+            app_root: sdk_directory,
+            sub_bin:  ''
+          }].map { |i| i.merge({ expected: APP_NAME }) }
+        end
+
+        # location of SDK files
+        def sdk_directory=(v)
+          Log.log.debug{"sdk_directory=#{v}"}
+          @sdk_dir = v
+          sdk_directory
+        end
+
+        # @return the path to folder where SDK is installed
+        def sdk_directory
+          Aspera.assert(!@sdk_dir.nil?){'SDK path was not initialized'}
+          FileUtils.mkdir_p(@sdk_dir)
+          @sdk_dir
+        end
+
+        def transferd_path
+          return File.join(sdk_directory, Environment.exe_file('asperatransferd')) # cspell:disable-line
+        end
+
+        # Well, the port number is only in log file
+        def daemon_port_from_log(log_file)
+          result = nil
+          # if port is zero, a dynamic port was created, get it
+          File.open(log_file, 'r') do |file|
+            file.each_line do |line|
+              # Well, it's tricky to depend on log
+              if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
+                result = m[2].to_i
+                # no "break" , need to read last matching log line
+              end
+            end
+          end
+          raise 'Port not found in daemon logs' if result.nil?
+          Log.log.debug{"Got port #{result} from log"}
+          return result
+        end
+      end
+    end
+  end
+end