aspera-cli 4.27.3 → 4.27.5

data/lib/aspera/oauth/base.rb

@@ -83,7 +83,7 @@ module Aspera
       # @return [String] The bearer token
       def token(cache: true, refresh: false)
         # Get token info from cache (or nil), decoded with date and expiration status
-        token_info = Factory.instance.get_token_info(@token_cache_id) if cache
+        token_info = Factory.instance.get_token_info(@token_cache_id, token_field: @token_field) if cache
         token_data = nil
         unless token_info.nil?
           token_data = token_info[:data]
@@ -130,6 +130,7 @@ module Aspera
           json_data = create_token.body
           token_data = JSON.parse(json_data)
           Factory.instance.persist_mgr.put(@token_cache_id, json_data)
+          Factory.instance.get_token_info(@token_cache_id, token_field: @token_field)
         end
         Aspera.assert(token_data.key?(@token_field)) { "API error: No such field in answer: #{@token_field}" } unless token_data.nil?
         # Ok we shall have a token here

data/lib/aspera/oauth/factory.rb

@@ -121,7 +121,8 @@ module Aspera
         data.each.map do |k, v|
           info = {id: k}
           begin; info.merge!(JSON.parse(v)); rescue StandardError; nil; end
-          d = decode_token(info.delete(TOKEN_FIELD))
+          token_val = info.delete(TOKEN_FIELD) || info.delete('token')
+          d = decode_token(token_val)
           info.merge(d) if d
           info
         end
@@ -129,13 +130,15 @@ module Aspera
 
       # Get token information from cache
       # @param id [String] identifier of token
+      # @param token_field [String, nil] Field containing the token, or nil to auto-detect
       # @return [Hash] token internal information , including Date object for `expiration_date`
-      def get_token_info(id)
+      def get_token_info(id, token_field: nil)
         token_raw_string = persist_mgr.get(id)
         return if token_raw_string.nil?
         token_data = JSON.parse(token_raw_string)
         Aspera.assert_type(token_data, Hash)
-        decoded_token = decode_token(token_data[TOKEN_FIELD])
+        token_value = token_data[token_field] || token_data[TOKEN_FIELD] || token_data['token']
+        decoded_token = decode_token(token_value)
         info = {data: token_data}
         if decoded_token.is_a?(Hash)
           info[:decoded] = decoded_token

data/lib/aspera/oauth/json_credentials.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'aspera/oauth/base'
+require 'digest'
+
+module Aspera
+  module OAuth
+    # Token creator using JSON credentials (e.g., username/password or apikey) sent to a login endpoint
+    class JsonCredentials < Base
+      # @param json           [Hash] Body parameters to send as JSON (e.g. {username:, password:} or {apikey:})
+      # @param generic_params [Hash] Generic parameters for OAuth::Base
+      def initialize(
+        json:,
+        **generic_params
+      )
+        cache_id = json[:username] || Digest::SHA256.hexdigest(json[:apikey] || json.to_s)[0..23]
+        super(**generic_params, cache_ids: [cache_id])
+        @body = json
+      end
+
+      def create_token
+        api.call(
+          operation:    'POST',
+          subpath:      path_token,
+          content_type: Mime::JSON,
+          body:         @body,
+          headers:      {'Accept' => Mime::JSON},
+          ret:          :resp
+        )
+      end
+    end
+    Factory.instance.register_token_creator(JsonCredentials)
+  end
+end

data/lib/aspera/oauth/jwt.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'aspera/oauth/base'
+require 'aspera/environment'
 require 'aspera/assert'
 require 'securerandom'
 require 'openssl'
@@ -17,10 +18,8 @@ module Aspera
       class << self
         def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
           priv_key = OpenSSL::PKey::RSA.new(length)
-          File.write(path, priv_key.to_s)
-          File.write("#{path}.pub", priv_key.public_key.to_s)
-          Environment.restrict_file_access(path)
-          Environment.restrict_file_access("#{path}.pub")
+          Environment.write_file_restricted(path, force: true) { priv_key.to_s }
+          Environment.write_file_restricted("#{path}.pub", force: true) { priv_key.public_key.to_s }
           nil
         end
       end

data/lib/aspera/oauth.rb

@@ -6,3 +6,4 @@ require 'aspera/oauth/generic'
 require 'aspera/oauth/jwt'
 require 'aspera/oauth/web'
 require 'aspera/oauth/url_json'
+require 'aspera/oauth/json_credentials'

data/lib/aspera/persistency_folder.rb

@@ -44,9 +44,7 @@ module Aspera
       Aspera.assert_type(value, String)
       persist_filepath = id_to_filepath(object_id)
       Log.log.debug { "persistency saving: #{persist_filepath}" }
-      FileUtils.rm_f(persist_filepath)
-      File.write(persist_filepath, value)
-      Environment.restrict_file_access(persist_filepath)
+      Environment.write_file_restricted(persist_filepath, force: true) { value }
       @cache[object_id] = value
       nil
     end

data/lib/aspera/preview/generator.rb

@@ -4,9 +4,9 @@
 # spellchecker:ignore soffice pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
 # spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos unoconv optipng reencode conv transframes
 
-require 'aspera/preview/options'
 require 'aspera/preview/utils'
 require 'aspera/preview/file_types'
+require 'aspera/schema/registry'
 require 'aspera/log'
 require 'aspera/assert'
 
@@ -20,6 +20,9 @@ module Aspera
       # List of valid ffmpeg option keys for reencode configuration.
       FFMPEG_OPTIONS_LIST = %w[in out].freeze
 
+      # Generator options, members defined in `options.schema.yaml`.
+      Options = Struct.new(*Schema::Registry.instance.reader(Schema::Registry::PREVIEW_OPTIONS).current['properties'].keys.map(&:to_sym))
+
       # CLI needs to know conversion type to know if need skip it.
       # One of CONVERSION_TYPES.
       attr_reader :conversion_type, :destination

data/lib/aspera/preview/options.schema.yaml

@@ -0,0 +1,119 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/preview/options.schema.yaml
+title: Preview generator options
+description: >-
+  Options of the preview generator: tools and parameters used to generate PNG thumbnails and MP4 video previews.
+$comment: >-
+  Used to declare the options of the `preview` plugin: values of `enum` properties are `Symbol`.
+  Scaling uses `ffmpeg` scale filter arguments, see: https://trac.ffmpeg.org/wiki/Scaling
+  (`iw`/`ih`: input width/height, `-x`: keep aspect ratio, with a value multiple of `x`).
+type: object
+additionalProperties: false
+properties:
+  office_conversion:
+    description: 'PNG thumbnail of office document: conversion tool.'
+    type: string
+    enum: [soffice, unoconv]
+    default: soffice
+  max_size:
+    description: 'Maximum size (in bytes) of a preview file: a warning is logged if exceeded.'
+    type: integer
+    minimum: 1
+    default: 16777216
+  thumb_vid_scale:
+    description: 'PNG thumbnail of video: frame size, as `ffmpeg` scale filter argument.'
+    type: string
+    default: '-1:min(ih,100)'
+  thumb_vid_fraction:
+    description: 'PNG thumbnail of video (`fixed`): position of the snapshot, as a fraction of the video duration.'
+    type: number
+    minimum: 0
+    maximum: 1
+    default: 0.1
+  thumb_img_size:
+    description: 'PNG thumbnail of image, PDF, office document or text: size (in pixels).'
+    type: integer
+    minimum: 1
+    default: 800
+  thumb_text_font:
+    description: 'PNG thumbnail of text: font name, as listed by `magick identify -list font`.'
+    type: string
+    default: Courier
+  video_conversion:
+    description: >-
+      MP4 video preview: generation method.
+      `reencode`: re-encode the beginning of the video,
+      `blend`: key frames with transitions,
+      `clips`: concatenation of short clips.
+    type: string
+    enum: [reencode, blend, clips]
+    default: reencode
+  video_png_conv:
+    description: >-
+      PNG thumbnail of video: generation method.
+      `fixed`: single frame,
+      `animated`: animated PNG.
+    type: string
+    enum: [fixed, animated]
+    default: fixed
+  video_scale:
+    description: 'MP4 video preview: frame size, as `ffmpeg` scale filter argument.'
+    type: string
+    default: 'min(iw,360):-2'
+  video_start_sec:
+    description: 'Video: start offset (in seconds) of MP4 video preview and of `animated` PNG thumbnail.'
+    type: integer
+    minimum: 0
+    default: 10
+  video_codec:
+    description: >-
+      MP4 video preview (`reencode`): `ffmpeg` video codec, e.g. `libx264`, `h264_videotoolbox`, `h264_nvenc`.
+      Default: first available H.264 encoder.
+    type: string
+  reencode_ffmpeg:
+    description: >-
+      MP4 video preview (`reencode`): `ffmpeg` options replacing the default ones.
+    type: object
+    additionalProperties: false
+    properties:
+      in:
+        description: Input options.
+        type: array
+        items:
+          type: [string, number]
+      out:
+        description: Output options.
+        type: array
+        items:
+          type: [string, number]
+    default: {}
+  blend_keyframes:
+    description: 'MP4 video preview (`blend`): number of key frames.'
+    type: integer
+    minimum: 1
+    default: 30
+  blend_pauseframes:
+    description: 'MP4 video preview (`blend`): number of pause frames (repetitions of each key frame).'
+    type: integer
+    minimum: 0
+    default: 3
+  blend_transframes:
+    description: 'MP4 video preview (`blend`): number of transition frames between key frames.'
+    type: integer
+    minimum: 0
+    default: 5
+  blend_fps:
+    description: 'MP4 video preview (`blend`): frames per second.'
+    type: integer
+    minimum: 1
+    default: 15
+  clips_count:
+    description: 'MP4 video preview (`clips`): number of clips.'
+    type: integer
+    minimum: 1
+    default: 5
+  clips_length:
+    description: 'Video: length (in seconds) of each clip of MP4 video preview (`clips`), and of `animated` PNG thumbnail.'
+    type: integer
+    minimum: 1
+    default: 5

data/lib/aspera/rest/aspera_errors.rb

@@ -2,6 +2,7 @@
 
 require 'aspera/rest/error_analyzer'
 require 'aspera/log'
+require 'xmlsimple'
 
 module Aspera
   module Rest
@@ -45,6 +46,17 @@ module Aspera
           ErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
             next if context[:response].code.start_with?('2')
             data = context[:data]
+            if data.is_a?(String) && data.start_with?('<?xml')
+              parsed = XmlSimple.xml_in(data, {'ForceArray' => false}) rescue nil
+              if parsed.is_a?(Hash) && parsed['content'].is_a?(String)
+                ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
+                parsed.each do |k, v|
+                  next if k.eql?('content')
+                  ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
+                end
+              end
+              next
+            end
             next unless data.is_a?(Hash)
             work_order = data['work_order']
             next unless work_order.is_a?(Hash)

data/lib/aspera/rest/client.rb

@@ -401,31 +401,39 @@ module Aspera
         Log.log.debug { "saving to: #{target_file}" }
         written_size = 0
         session_id = SecureRandom.uuid.freeze
-        Parameters.instance.progress_bar&.event(:session_start, session_id: session_id)
-        Parameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
+        progress = Parameters.instance.progress_bar
+        progress&.event(:session_start, session_id: session_id)
+        progress&.event(:session_size, session_id: session_id, info: total_size) if total_size
         limiter = TimerLimiter.new(0.5)
-        if target_file.respond_to?(:write)
-          # IO object: stream directly into it
-          response.read_body do |fragment|
-            target_file.write(fragment)
-            written_size += fragment.length
-            Parameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
-          end
-        else
-          # file path: download to partial name first, then rename atomically
-          target_file_tmp = "#{target_file}#{Parameters.instance.download_partial_suffix}"
-          FileUtils.mkdir_p(File.dirname(target_file_tmp))
-          File.open(target_file_tmp, 'wb') do |file|
+        success = false
+        begin
+          if target_file.respond_to?(:write)
+            # IO object: stream directly into it
             response.read_body do |fragment|
-              file.write(fragment)
+              target_file.write(fragment)
               written_size += fragment.length
-              Parameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
+              progress&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
+            end
+          else
+            # file path: download to partial name first, then rename atomically
+            target_file_tmp = "#{target_file}#{Parameters.instance.download_partial_suffix}"
+            FileUtils.mkdir_p(File.dirname(target_file_tmp))
+            File.open(target_file_tmp, 'wb') do |file|
+              response.read_body do |fragment|
+                file.write(fragment)
+                written_size += fragment.length
+                progress&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
+              end
             end
+            File.rename(target_file_tmp, target_file)
           end
-          File.rename(target_file_tmp, target_file)
+          success = true
+        ensure
+          # Last progress may have been skipped by limiter
+          progress&.event(:transfer, session_id: session_id, info: written_size)
+          progress&.event(:session_end, session_id: session_id)
+          progress&.event(:end, info: success)
         end
-        Parameters.instance.progress_bar&.event(:session_end, session_id: session_id)
-        Parameters.instance.progress_bar&.event(:end)
         true
       end
 
@@ -439,6 +447,7 @@ module Aspera
         # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
         node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
         result_data = result_data[0..node_api_bug] if node_api_bug
+        result_mime = 'application/xml' if result_data.is_a?(String) && result_data.start_with?('<?xml')
         result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
         Log.dump(:result_data, result_data)
         result_data

data/lib/aspera/rest/list.rb

@@ -70,9 +70,11 @@ module Aspera
         Log.dump(:query, query)
         result = []
         offset = 0
-        max_items = query.delete(MAX_ITEMS)
-        remain_pages = query.delete(MAX_PAGES)
-        # Merge default parameters, by default 100 per page
+        # Special parameters are not sent to the API (the caller's query is not modified)
+        query = query.dup
+        max_items = query.delete(MAX_ITEMS)&.to_i
+        remain_pages = query.delete(MAX_PAGES)&.to_i
+        # Merge default parameters
         query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
         total_count = nil
         call_args = {
@@ -84,18 +86,22 @@ module Aspera
         loop do
           query['offset'] = offset
           page_result = call(**call_args)
-          Aspera.assert_type(page_result[items_key], Array)
-          result.concat(page_result[items_key])
+          page_items = page_result[items_key]
+          Aspera.assert_type(page_items, Array)
+          result.concat(page_items)
           # Reach the limit set by user ?
           if !max_items.nil? && (result.length >= max_items)
             result = result.slice(0, max_items)
             break
           end
           total_count ||= page_result['total_count']
-          break if result.length >= total_count
+          # An empty page ends the list, even if the total count is not reached (e.g. items deleted meanwhile)
+          break if page_items.empty?
+          # Without total count, a partial page is the last one
+          break if total_count.nil? ? page_items.length < query['limit'].to_i : result.length >= total_count
           remain_pages -= 1 unless remain_pages.nil?
-          break if remain_pages == 0
-          offset += page_result[items_key].length
+          break if !remain_pages.nil? && remain_pages <= 0
+          offset += page_items.length
           Parameters.instance.spinner_cb.call("#{result.length} / #{total_count || '?'}")
         end
         Parameters.instance.spinner_cb.call(action: :success)

data/lib/aspera/rest/parameters.rb

@@ -32,7 +32,7 @@ module Aspera
       attr_accessor :session_cb
       # @return [Object, nil] Progress bar, receives `event` calls during download
       attr_accessor :progress_bar
-      # @return [Proc, nil] Called with `(title = nil, action: :spin)` to display progress of long operations
+      # @return [Proc] Called with `(title = nil, action: :spin)` to display progress of long operations (default: no display)
       attr_accessor :spinner_cb
 
       private
@@ -48,7 +48,7 @@ module Aspera
         @retry_sleep = 4
         @session_cb = nil
         @progress_bar = nil
-        @spinner_cb = nil
+        @spinner_cb = ->(_title = nil, action: :spin) {}
       end
     end
   end

data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml

@@ -3840,12 +3840,6 @@ paths:
           required: false
           schema:
             type: boolean
-        - name: get_package_download_link
-          in: query
-          description: Specifies whether to return the package download context to download the package as an external user.
-          required: false
-          schema:
-            type: boolean
       responses:
         '200':
           description: ' - OK - Successful response'
@@ -30208,13 +30202,6 @@ components:
             - eyJyZXNvdXJjZSI6InBhY2thZ2VzIiwidHlwZSI6ImV4dGVybmFsX3ZpZXdfcGFja2FnZSIsImlkIjoiMzAiLCJwYXNzY29kZSI6IjA2MzBkZDIzLTA3YTAtNGExYi04N2IzLWM3YjZkMGViNjE5NiIsInBhY2thZ2VfaWQiOiIzMCJ9
           maxLength: 255
           minLength: 1
-        pkg_download_context:
-          description: The package download context value used for navigating to the package download view for external users. Only included in responses when `get_package_download_link` is set to `true` and the package in question was not sent to a shared inbox or workgroup.
-          type: string
-          examples:
-            - eyJyZXNvdXJjZSI6InBhY2thZ2VzIiwidHlwZSI6ImV4dGVybmFsX3ZpZXdfcGFja2FnZSIsImlkIjoiMzAiLCJwYXNzY29kZSI6IjA2MzBkZDIzLTA3YTAtNGExYi04N2IzLWM3YjZkMGViNjE5NiIsInBhY2thZ2VfaWQiOiIzMCJ9
-          maxLength: 255
-          minLength: 1
     PutPackage:
       description: Update a package
       type: object
@@ -32779,13 +32766,6 @@ components:
                   - eyJyZXNvdXJjZSI6InBhY2thZ2VzIiwidHlwZSI6ImV4dGVybmFsX3ZpZXdfcGFja2FnZSIsImlkIjoiMzAiLCJwYXNzY29kZSI6IjA2MzBkZDIzLTA3YTAtNGExYi04N2IzLWM3YjZkMGViNjE5NiIsInBhY2thZ2VfaWQiOiIzMCJ9
                 maxLength: 255
                 minLength: 1
-              pkg_download_context:
-                description: The package download context value used for navigating to the package download view for external users. Only included in responses when `get_package_download_link` is set to `true` and the package in question was not sent to a shared inbox or workgroup.
-                type: string
-                examples:
-                  - eyJyZXNvdXJjZSI6InBhY2thZ2VzIiwidHlwZSI6ImV4dGVybmFsX3ZpZXdfcGFja2FnZSIsImlkIjoiMzAiLCJwYXNzY29kZSI6IjA2MzBkZDIzLTA3YTAtNGExYi04N2IzLWM3YjZkMGViNjE5NiIsInBhY2thZ2VfaWQiOiIzMCJ9
-                maxLength: 255
-                minLength: 1
               relays_count:
                 description: Package relays count
                 type: object