aspera-cli 4.14.0 → 4.15.0

data/lib/aspera/oauth.rb

@@ -24,18 +24,23 @@ module Aspera
     # OAuth methods supported by default
     STD_AUTH_TYPES = %i[web jwt].freeze
 
-    # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
-    JWT_ACCEPTED_OFFSET_SEC = 300
-    # one hour validity (TODO: configurable?)
-    JWT_EXPIRY_OFFSET_SEC = 3600
-    # tokens older than 30 minutes will be discarded from cache
-    TOKEN_CACHE_EXPIRY_SEC = 1800
-    # tokens valid for less than this duration will be regenerated
-    TOKEN_EXPIRATION_GUARD_SEC = 120
+    @@globals = { # rubocop:disable Style/ClassVars
+      # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
+      jwt_accepted_offset_sec:    300,
+      # one hour validity (TODO: configurable?)
+      jwt_expiry_offset_sec:      3600,
+      # tokens older than 30 minutes will be discarded from cache
+      token_cache_expiry_sec:     1800,
+      # tokens valid for less than this duration will be regenerated
+      token_expiration_guard_sec: 120
+    }
+
     # a prefix for persistency of tokens (simplify garbage collect)
     PERSIST_CATEGORY_TOKEN = 'token'
+    # prefix for bearer token when in header
+    BEARER_PREFIX = 'Bearer '
 
-    private_constant :JWT_ACCEPTED_OFFSET_SEC, :JWT_EXPIRY_OFFSET_SEC, :TOKEN_CACHE_EXPIRY_SEC, :PERSIST_CATEGORY_TOKEN, :TOKEN_EXPIRATION_GUARD_SEC
+    private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
 
     # persistency manager
     @persist = nil
@@ -45,10 +50,23 @@ module Aspera
     @id_handlers = {}
 
     class << self
+      def bearer_build(token)
+        return BEARER_PREFIX + token
+      end
+
+      def bearer_extract(token)
+        raise 'not a bearer token, wrong prefix' unless bearer?(token)
+        return token[BEARER_PREFIX.length..-1]
+      end
+
+      def bearer?(token)
+        return token.start_with?(BEARER_PREFIX)
+      end
+
       def persist_mgr=(manager)
         @persist = manager
         # cleanup expired tokens
-        @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, TOKEN_CACHE_EXPIRY_SEC)
+        @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @@globals[:token_cache_expiry_sec])
       end
 
       def persist_mgr
@@ -153,9 +171,9 @@ module Aspera
       Log.log.info{"seconds=#{seconds_since_epoch}"}
       raise 'missing JWT payload' unless oauth.specific_parameters[:payload].is_a?(Hash)
       jwt_payload = {
-        exp: seconds_since_epoch + JWT_EXPIRY_OFFSET_SEC, # expiration time
-        nbf: seconds_since_epoch - JWT_ACCEPTED_OFFSET_SEC, # not before
-        iat: seconds_since_epoch - JWT_ACCEPTED_OFFSET_SEC + 1, # issued at (we tell a little in the past so that server always accepts)
+        exp: seconds_since_epoch + @@globals[:jwt_expiry_offset_sec], # expiration time
+        nbf: seconds_since_epoch - @@globals[:jwt_accepted_offset_sec], # not before
+        iat: seconds_since_epoch - @@globals[:jwt_accepted_offset_sec] + 1, # issued at (we tell a little in the past so that server always accepts)
         jti: SecureRandom.uuid # JWT id
       }.merge(oauth.specific_parameters[:payload])
       Log.log.debug{"JWT jwt_payload=[#{jwt_payload}]"}
@@ -213,8 +231,8 @@ module Aspera
       @generic_parameters.delete(:base_url)
       @generic_parameters.delete(:auth)
       @generic_parameters.delete(@generic_parameters[:grant_method])
-      Log.dump(:generic_parameters, @generic_parameters)
-      Log.dump(:specific_parameters, @specific_parameters)
+      Log.log.debug{Log.dump(:generic_parameters, @generic_parameters)}
+      Log.log.debug{Log.dump(:specific_parameters, @specific_parameters)}
     end
 
     public
@@ -259,14 +277,14 @@ module Aspera
       # `direct` agent is equipped with refresh code
       if !use_refresh_token && !token_data.nil?
         decoded_token = self.class.decode_token(token_data[@generic_parameters[:token_field]])
-        Log.dump('decoded_token', decoded_token) unless decoded_token.nil?
+        Log.log.debug{Log.dump('decoded_token', decoded_token)} unless decoded_token.nil?
         if decoded_token.is_a?(Hash)
           expires_at_sec =
             if    decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
             elsif decoded_token['exp'].is_a?(Integer)       then Time.at(decoded_token['exp'])
             end
           # force refresh if we see a token too close from expiration
-          use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < TOKEN_EXPIRATION_GUARD_SEC
+          use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < @@globals[:token_expiration_guard_sec]
           Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
         end
       end
@@ -306,7 +324,7 @@ module Aspera
       end # if ! in_cache
       raise "API error: No such field in answer: #{@generic_parameters[:token_field]}" unless token_data.key?(@generic_parameters[:token_field])
       # ok we shall have a token here
-      return 'Bearer ' + token_data[@generic_parameters[:token_field]]
+      return self.class.bearer_build(token_data[@generic_parameters[:token_field]])
     end
   end # OAuth
 end # Aspera

data/lib/aspera/open_application.rb

@@ -27,7 +27,7 @@ module Aspera
       def uri_graphical(uri)
         case Aspera::Environment.os
         when Aspera::Environment::OS_X       then return system('open', uri.to_s)
-        when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', '"' + uri.to_s + '"')
+        when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', %Q{"#{uri}"})
         when Aspera::Environment::OS_LINUX   then return system('xdg-open', uri.to_s)
         else
           raise "no graphical open method for #{Aspera::Environment.os}"
@@ -38,7 +38,7 @@ module Aspera
         if ENV.key?('EDITOR')
           system(ENV['EDITOR'], file_path.to_s)
         elsif Aspera::Environment.os.eql?(Aspera::Environment::OS_WINDOWS)
-          system('notepad.exe', '"' + file_path.to_s + '"')
+          system('notepad.exe', %Q{"#{file_path}"})
         else
           uri_graphical(file_path.to_s)
         end
@@ -59,9 +59,9 @@ module Aspera
       when :text
         case the_url.to_s
         when /^http/
-          puts "USER ACTION: please enter this url in a browser:\n" + the_url.to_s.red + "\n"
+          puts "USER ACTION: please enter this url in a browser:\n#{the_url.to_s.red}\n"
         else
-          puts "USER ACTION: open this:\n" + the_url.to_s.red + "\n"
+          puts "USER ACTION: open this:\n#{the_url.to_s.red}\n"
         end
       else
         raise StandardError, "unsupported url open method: #{@url_method}"

data/lib/aspera/persistency_folder.rb

@@ -37,7 +37,7 @@ module Aspera
       raise 'value: only String supported' unless value.is_a?(String)
       persist_filepath = id_to_filepath(object_id)
       Log.log.debug{"persistency saving: #{persist_filepath}"}
-      File.delete(persist_filepath) if File.exist?(persist_filepath)
+      FileUtils.rm_f(persist_filepath)
       File.write(persist_filepath, value)
       Environment.restrict_file_access(persist_filepath)
       @cache[object_id] = value
@@ -46,7 +46,7 @@ module Aspera
     def delete(object_id)
       persist_filepath = id_to_filepath(object_id)
       Log.log.debug{"persistency deleting: #{persist_filepath}"}
-      File.delete(persist_filepath) if File.exist?(persist_filepath)
+      FileUtils.rm_f(persist_filepath)
       @cache.delete(object_id)
     end
 

data/lib/aspera/preview/file_types.rb

@@ -281,7 +281,7 @@ module Aspera
 
       # use mime magic to find mime type based on file content (magic numbers)
       def mime_from_file(filepath)
-        # moved here, as mimemagic can cause installation issues
+        # moved here, as `mimemagic` can cause installation issues
         require 'mimemagic'
         require 'mimemagic/version'
         require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
@@ -297,9 +297,10 @@ module Aspera
         return detected_mime
       end
 
-      # return file type, one of enum CONVERSION_TYPES
       # @param filepath [String] full path to file
       # @param mimetype [String] provided by node api
+      # @return file type, one of enum CONVERSION_TYPES
+      # @raise [RuntimeError] if no conversion type found
       def conversion_type(filepath, mimetype)
         Log.log.debug{"conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})"}
         # 1- get type from provided mime type, using local mapping
@@ -322,6 +323,7 @@ module Aspera
         # 3- else, from extensions, using local mapping
         extension = File.extname(filepath.downcase)[1..-1]
         conversion_type = SUPPORTED_EXTENSIONS[extension] if conversion_type.nil?
+        raise "no conversion type found for extension #{extension}" if conversion_type.nil?
         Log.log.debug{"conversion_type(#{extension}): #{conversion_type.class.name} [#{conversion_type}]"}
         return conversion_type
       end

data/lib/aspera/preview/generator.rb

@@ -2,7 +2,7 @@
 
 # ffmpeg options:
 # spellchecker:ignore pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
-# spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos
+# spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos unoconv optipng reencode conv transframes
 
 require 'open3'
 require 'aspera/preview/options'
@@ -19,6 +19,7 @@ module Aspera
       FFMPEG_OPTIONS_LIST = %w[in out].freeze
 
       # CLI needs to know conversion type to know if need skip it
+      # one of CONVERSION_TYPES
       attr_reader :conversion_type
 
       # @param src source file path
@@ -32,55 +33,40 @@ module Aspera
       #   -> preview_format is one of Generator::PREVIEW_FORMATS
       # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
       #  -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
-      def initialize(options, src, dst, main_temp_dir, api_mime_type)
-        @options = options
+      def initialize(src, dst, options, main_temp_dir, api_mime_type)
         @source_file_path = src
         @destination_file_path = dst
+        @options = options
         @temp_folder = File.join(main_temp_dir, @source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
         # extract preview format from extension of target file
-        @preview_format_symb = File.extname(@destination_file_path).gsub(/^\./, '').to_sym
-        @conversion_type = FileTypes.instance.conversion_type(@source_file_path, api_mime_type)
-      end
-
-      # name of processing method in this object
-      # combination of: conversion type and output format (and video_conversion for video)
-      def processing_method_symb
-        name = "convert_#{@conversion_type}_to_#{@preview_format_symb}"
-        if @conversion_type.eql?(:video)
-          case @preview_format_symb
+        @preview_format_sym = File.extname(@destination_file_path).gsub(/^\./, '').to_sym
+        conversion_type = FileTypes.instance.conversion_type(@source_file_path, api_mime_type)
+        @processing_method = "convert_#{conversion_type}_to_#{@preview_format_sym}"
+        if conversion_type.eql?(:video)
+          case @preview_format_sym
           when :mp4
-            name = "#{name}_using_#{@options.video_conversion}"
+            @processing_method = "#{@processing_method}_using_#{@options.video_conversion}"
           when :png
-            name = "#{name}_using_#{@options.video_png_conv}"
+            @processing_method = "#{@processing_method}_using_#{@options.video_png_conv}"
           end
         end
-        Log.log.debug{"method: #{name}"}
-        return name.to_sym
-      end
-
-      # @return true if conversion is supported.
-      # for instance, plaintext to mp4 is not supported
-      def supported?
-        return false if @conversion_type.nil?
-        return respond_to?(processing_method_symb, true)
+        @processing_method = @processing_method.to_sym
+        Log.log.debug{"method: #{@processing_method}"}
+        raise "no processing know for #{conversion_type} -> #{@preview_format_sym}" unless respond_to?(@processing_method, true)
       end
 
       # create preview as specified in constructor
       def generate
-        raise 'could not detect type of file' if @conversion_type.nil?
-        method_symb = processing_method_symb
-        Log.log.info{"#{@source_file_path}->#{@destination_file_path} (#{method_symb})"}
+        Log.log.info{"#{@source_file_path}->#{@destination_file_path} (#{@processing_method})"}
         begin
-          send(method_symb)
+          send(@processing_method)
           # check that generated size does not exceed maximum
           result_size = File.size(@destination_file_path)
-          if result_size > @options.max_size
-            Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"}
-          end
+          Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"} if result_size > @options.max_size
         rescue StandardError => e
           Log.log.error{"Ignoring: #{e.message}"}
           Log.log.debug(e.backtrace.join("\n").red)
-          FileUtils.cp(File.expand_path(@preview_format_symb.eql?(:mp4) ? 'video_error.png' : 'image_error.png', File.dirname(__FILE__)), @destination_file_path)
+          FileUtils.cp(File.expand_path(@preview_format_sym.eql?(:mp4) ? 'video_error.png' : 'image_error.png', File.dirname(__FILE__)), @destination_file_path)
         ensure
           FileUtils.rm_rf(@temp_folder)
         end
@@ -134,8 +120,8 @@ module Aspera
       # generate n clips starting at offset
       def convert_video_to_mp4_using_clips
         p_duration = Utils.video_get_duration(@source_file_path)
-        filelist = File.join(this_tmpdir, 'clip_files.txt')
-        File.open(filelist, 'w+') do |f|
+        file_list_file = File.join(this_tmpdir, 'clip_files.txt')
+        File.open(file_list_file, 'w+') do |f|
           1.upto(@options.clips_count.to_i) do |i|
             offset_seconds = get_offset(p_duration, @options.video_start_sec.to_i, @options.clips_count.to_i, i)
             tmp_file_name = format('clip%04d.mp4', i)
@@ -153,10 +139,11 @@ module Aspera
         end
         # concat clips
         Utils.ffmpeg(
-          in_f: filelist,
+          in_f: file_list_file,
           in_p: ['-f', 'concat'],
           out_f: @destination_file_path,
           out_p: ['-codec', 'copy'])
+        File.delete(file_list_file)
       end
 
       # do a simple re-encoding

data/lib/aspera/preview/options.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# cspell:ignore reencode conv transframes pauseframes
+
 module Aspera
   module Preview
     # generator options. Used as parameter to preview generator object.

data/lib/aspera/preview/terminal.rb

@@ -7,25 +7,34 @@ require 'rainbow'
 require 'io/console'
 module Aspera
   module Preview
-    # Generates a string that can display an image in a terminal
+    # Display a picture in the terminal, either using coloured characters or iTerm2
     class Terminal
-      # quantum depth is 8 or 16: convert xc: -format "%q" info:
       # Rainbow only supports 8-bit colors
+      # quantum depth is 8 or 16, see: `convert xc: -format "%q" info:`
       SHIFT_FOR_8_BIT = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
-      ITERM_NAMES = %w[iTerm WezTerm mintty].freeze
+      # env vars to detect terminal type
       TERM_ENV_VARS = %w[TERM_PROGRAM LC_TERMINAL].freeze
-      private_constant :SHIFT_FOR_8_BIT, :ITERM_NAMES, :TERM_ENV_VARS
+      # terminal names that support iTerm2 image display
+      ITERM_NAMES = %w[iTerm WezTerm mintty].freeze
+      # TODO: retrieve terminal font ratio using some termcap ?
+      # ratio = font height / font width
+      DEFAULT_FONT_RATIO = 32.0 / 14.0
+      private_constant :SHIFT_FOR_8_BIT, :TERM_ENV_VARS, :ITERM_NAMES, :DEFAULT_FONT_RATIO
       class << self
-        def build(blob, reserved_lines: 0, double_precision: true)
-          return iterm_display_image(blob) if iterm_supported?
+        # @return [String] the image as text, or the iTerm2 escape sequence
+        # @param blob [String] the image as a binary string
+        # @param reserve [Integer] number of lines to reserve for other text than the image
+        # @param text [Boolean] true to display the image as text, false to use iTerm2
+        # @param double [Boolean] true to use colors on half lines, false to use colors on full lines
+        # @param font_ratio [Float] ratio = font height / font width
+        def build(blob, reserve: 3, text: false, double: true, font_ratio: DEFAULT_FONT_RATIO)
+          return iterm_display_image(blob) if iterm_supported? && !text
           image = Magick::ImageList.new.from_blob(blob)
           (term_rows, term_columns) = IO.console.winsize
-          term_rows -= reserved_lines
+          term_rows -= reserve
           # compute scaling to fit terminal
           fit_term_ratio = [term_rows / image.rows.to_f, term_columns / image.columns.to_f].min
-          # TODO: retrieve terminal font ratio using some termcap ?
-          font_ratio = 1.7
-          height_ratio = double_precision ? 2.0 : 1.0
+          height_ratio = double ? 2.0 : 1.0
           image = image.scale((image.columns * fit_term_ratio * font_ratio).to_i, (image.rows * fit_term_ratio * height_ratio).to_i)
           # get all pixel colors, adjusted for Rainbow
           pixel_colors = []
@@ -39,10 +48,10 @@ module Aspera
           # now generate text
           text_pixels = []
           pixel_colors.each_with_index do |row_data, row|
-            next if double_precision && row.odd?
+            next if double && (row.odd? || row.eql?(pixel_colors.length - 1))
             row_data.each_with_index do |pixel_rgb, col|
               text_pixels.push("\n") if col.eql?(0) && !row.eql?(0)
-              if double_precision
+              if double
                 text_pixels.push(Rainbow('▄').background(pixel_rgb).foreground(pixel_colors[row + 1][col]))
               else
                 text_pixels.push(Rainbow(' ').background(pixel_rgb))
@@ -53,8 +62,10 @@ module Aspera
         end
 
         # display image in iTerm2
+        # https://iterm2.com/documentation-images.html
         def iterm_display_image(blob)
           # image = Magick::ImageList.new.from_blob(blob)
+          # parameters for iTerm2 image display
           arguments = {
             inline:              1,
             preserveAspectRatio: 1,
@@ -63,7 +74,7 @@ module Aspera
             # height:              image.rows
           }.map { |k, v| "#{k}=#{v}" }.join(';')
           # \a is BEL, \e is ESC : https://github.com/ruby/ruby/blob/master/doc/syntax/literals.rdoc#label-Strings
-          # https://iterm2.com/documentation-images.html
+          # escape sequence for iTerm2 image display
           return "\e]1337;File=#{arguments}:#{Base64.encode64(blob)}\a"
         end
 

data/lib/aspera/preview/utils.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+# cspell:ignore ffprobe optipng unoconv
 require 'English'
 require 'tmpdir'
 require 'fileutils'
@@ -11,18 +12,17 @@ module Aspera
     class Utils
       # from bash manual: meta-character need to be escaped
       BASH_SPECIAL_CHARACTERS = "|&;()<> \t#\n"
-      # shell exit code when command is not found
-      BASH_EXIT_NOT_FOUND = 127
       # external binaries used
       EXTERNAL_TOOLS = %i[ffmpeg ffprobe convert composite optipng unoconv].freeze
       TEMP_FORMAT = 'img%04d.jpg'
-      private_constant :BASH_SPECIAL_CHARACTERS, :BASH_EXIT_NOT_FOUND, :EXTERNAL_TOOLS, :TEMP_FORMAT
+      private_constant :BASH_SPECIAL_CHARACTERS, :EXTERNAL_TOOLS, :TEMP_FORMAT
 
       class << self
         # returns string with single quotes suitable for bash if there is any bash meta-character
         def shell_quote(argument)
           return argument unless argument.chars.any?{|c|BASH_SPECIAL_CHARACTERS.include?(c)}
-          return "'" + argument.gsub(/'/){|_s| "'\"'\"'"} + "'"
+          # surround with single quotes, and escape single quotes
+          return %Q{'#{argument.gsub("'"){|_s| %q{'"'"'}}}'}
         end
 
         # check that external tools can be executed
@@ -30,38 +30,31 @@ module Aspera
           tools_to_check = EXTERNAL_TOOLS.dup
           tools_to_check.delete(:unoconv) if skip_types.include?(:office)
           # Check for binaries
-          tools_to_check.each do |command_symb|
-            external_command(command_symb, ['-h'])
+          tools_to_check.each do |command_sym|
+            external_command(command_sym, ['-h'], log_error: false)
+          rescue Errno::ENOENT => e
+            raise "missing #{command_sym} binary: #{e}"
+          rescue
+            nil
           end
         end
 
         # execute external command
         # one could use "system", but we would need to redirect stdout/err
         # @return true if su
-        def external_command(command_symb, command_args)
-          raise "unexpected command #{command_symb}" unless EXTERNAL_TOOLS.include?(command_symb)
+        def external_command(command_sym, command_args, log_error: true)
+          raise "unexpected command #{command_sym}" unless EXTERNAL_TOOLS.include?(command_sym)
           # build command line, and quote special characters
-          command = command_args.clone.unshift(command_symb).map{|i| shell_quote(i.to_s)}.join(' ')
-          Log.log.debug{"cmd=#{command}".blue}
-          # capture3: only in ruby2+
-          if Open3.respond_to?(:capture3)
-            stdout, stderr, exit_status = Open3.capture3(command)
-          else
-            stderr = '<merged with stdout>'
-            stdout = %x(#{command} 2>&1)
-            exit_status = $CHILD_STATUS
-          end
-          if BASH_EXIT_NOT_FOUND.eql?(exit_status)
-            raise "Error: #{command_symb} is not in the PATH"
-          end
-          unless exit_status.success?
-            Log.log.error{"command line: #{command}"}
-            Log.log.error{"Error code: #{exit_status}"}
+          command_line = command_args.clone.unshift(command_sym).map{|i| shell_quote(i.to_s)}.join(' ')
+          Log.log.debug{"cmd=#{command_line}".blue}
+          stdout, stderr, status = Open3.capture3(command_line)
+          if log_error && !status.success?
+            Log.log.error{"status: #{status}"}
             Log.log.error{"stdout: #{stdout}"}
             Log.log.error{"stderr: #{stderr}"}
-            raise "#{command_symb} error #{exit_status}"
+            raise "#{command_sym} error #{status}"
           end
-          return {status: exit_status, stdout: stdout}
+          return {status: status, stdout: stdout}
         end
 
         def ffmpeg(a)