aspera-cli 4.27.5 → 4.28.0

data/lib/aspera/rest/aspera_errors.rb

@@ -1,71 +1,63 @@
 # frozen_string_literal: true
 
 require 'aspera/rest/error_analyzer'
-require 'aspera/log'
 require 'xmlsimple'
 
 module Aspera
   module Rest
-    # REST error handlers for various Aspera REST APIs
-    class AsperaErrors
-      class << self
-        # handlers should probably be defined by plugins for modularity
-        def register_handlers
-          Log.log.debug('registering Aspera REST error handlers')
-          # Faspex 4: both user_message and internal_message, and code 200
-          # example: missing meta data on package creation
-          ErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
-          ErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
-          ErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
-          ErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
-          ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
-          ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['failure'], always: true)
-          # AoC Automation
-          ErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
-          ErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, context|
-            next unless context[:data].is_a?(Hash) && context[:data]['errors'].is_a?(Hash)
-            # special for Shares: false positive ? (update global transfer_settings)
-            next if context[:data].key?('min_connect_version')
-            context[:data]['errors'].each do |k, v|
-              ErrorAnalyzer.add_error(context, type, "#{k}: #{v}")
-            end
-          end
-          # call to upload_setup and download_setup of node api
-          ErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, context|
-            next unless context[:data].is_a?(Hash)
-            d_t_s = context[:data]['transfer_specs']
-            next unless d_t_s.is_a?(Array)
-            d_t_s.each do |res|
-              r_err = res.dig(*%w[transfer_spec error]) || res['error']
-              next unless r_err.is_a?(Hash)
-              ErrorAnalyzer.add_error(context, type, r_err.values.join(': '))
-            end
-          end
-          ErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
-          ErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
-          ErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
-            next if context[:response].code.start_with?('2')
-            data = context[:data]
-            if data.is_a?(String) && data.start_with?('<?xml')
-              parsed = XmlSimple.xml_in(data, {'ForceArray' => false}) rescue nil
-              if parsed.is_a?(Hash) && parsed['content'].is_a?(String)
-                ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
-                parsed.each do |k, v|
-                  next if k.eql?('content')
-                  ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
-                end
-              end
-              next
-            end
-            next unless data.is_a?(Hash)
-            work_order = data['work_order']
-            next unless work_order.is_a?(Hash)
-            ErrorAnalyzer.add_error(context, type, work_order['statusDetails'])
-            data['missing_parameters']&.each do |param|
-              ErrorAnalyzer.add_error(context, type, "missing parameter: #{param}")
-            end
+    # REST error handlers for various Aspera REST APIs, registered on require
+    # handlers should probably be defined by plugins for modularity
+    # Faspex 4: both user_message and internal_message, and code 200
+    # example: missing meta data on package creation
+    ErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
+    ErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
+    ErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
+    ErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
+    ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
+    ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['failure'], always: true)
+    # AoC Automation
+    ErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
+    ErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, context|
+      next unless context[:data].is_a?(Hash) && context[:data]['errors'].is_a?(Hash)
+      # special for Shares: false positive ? (update global transfer_settings)
+      next if context[:data].key?('min_connect_version')
+      context[:data]['errors'].each do |k, v|
+        ErrorAnalyzer.add_error(context, type, "#{k}: #{v}")
+      end
+    end
+    # call to upload_setup and download_setup of node api
+    ErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, context|
+      next unless context[:data].is_a?(Hash)
+      d_t_s = context[:data]['transfer_specs']
+      next unless d_t_s.is_a?(Array)
+      d_t_s.each do |res|
+        r_err = res.dig(*%w[transfer_spec error]) || res['error']
+        next unless r_err.is_a?(Hash)
+        ErrorAnalyzer.add_error(context, type, r_err.values.join(': '))
+      end
+    end
+    ErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
+    ErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
+    ErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
+      next if context[:response].code.start_with?('2')
+      data = context[:data]
+      if data.is_a?(String) && data.start_with?('<?xml')
+        parsed = XmlSimple.xml_in(data, {'ForceArray' => false}) rescue nil
+        if parsed.is_a?(Hash) && parsed['content'].is_a?(String)
+          ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
+          parsed.each do |k, v|
+            next if k.eql?('content')
+            ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
           end
         end
+        next
+      end
+      next unless data.is_a?(Hash)
+      work_order = data['work_order']
+      next unless work_order.is_a?(Hash)
+      ErrorAnalyzer.add_error(context, type, work_order['statusDetails'])
+      data['missing_parameters']&.each do |param|
+        ErrorAnalyzer.add_error(context, type, "missing parameter: #{param}")
       end
     end
   end

data/lib/aspera/rest/client.rb

@@ -5,10 +5,9 @@ require 'aspera/rest/parameters'
 require 'aspera/rest/util'
 require 'aspera/rest/call_error'
 require 'aspera/rest/error_analyzer'
-require 'aspera/rest/aspera_errors'
 require 'aspera/log'
 require 'aspera/assert'
-require 'aspera/oauth'
+require 'aspera/oauth/client/factory'
 require 'aspera/hash_ext'
 require 'aspera/timer_limiter'
 require 'net/http'
@@ -126,11 +125,12 @@ module Aspera
       # @param subpath      [String] subpath of REST API
       # @param query        [Hash{String,Symbol => Object}] URL parameters
       # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
-      # @param body         [Hash, String, nil] Body parameters
+      # @param body         [Hash, Array, String, nil] Body parameters (see `build_request`)
       # @param headers      [Hash{String => String}] Additional headers (override Content-Type)
       # @param save_to      [String, Pathname, IO, nil] File path or IO object to save response body; progress bar is used when set
       # @param exception    [Boolean] Whether to raise an exception on HTTP error
       # @param ret          [Symbol] One of :data, :resp, :both - controls return value
+      # @param json_opts    [Hash] Options of `JSON.parse` for a JSON response, e.g. `{allow_duplicate_key: true}`
       # @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
       # @return [Net::HTTPResponse] When `ret` is :resp
       # @return [Hash] When `ret` is :data
@@ -144,7 +144,8 @@ module Aspera
         headers: nil,
         save_to: nil,
         exception: true,
-        ret: :data
+        ret: :data,
+        json_opts: {}
       )
         subpath = subpath.to_s if subpath.is_a?(Symbol)
         subpath = '' if subpath.nil?
@@ -185,7 +186,7 @@ module Aspera
           # TODO : related to mime type encoding ?
           # result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
           # Log.log.debug{"result: body=#{result_http.body}"}
-          result_data = parse_response(result_http, result_mime)
+          result_data = parse_response(result_http, mime_type: result_mime, json_opts: json_opts)
           ErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
           unless file_saved || save_to.nil?
             raise 'save_to: IO object requires a streaming response' if save_to.respond_to?(:write)
@@ -223,7 +224,8 @@ module Aspera
               save_to:      save_to,
               exception:    exception,
               headers:      headers,
-              ret:          ret
+              ret:          ret,
+              json_opts:    json_opts
             )
           end
           # raise exception if could not retry and not return error in result
@@ -339,8 +341,9 @@ module Aspera
       # @param operation    [String]                  HTTP operation (GET, POST, ...)
       # @param subpath      [String]                  Subpath of REST API
       # @param query        [Hash, String, Array, nil] Query of request
-      # @param content_type [String, nil]             One of Mime::JSON, Mime::WWW, Mime::TEXT, or `nil` for no body
-      # @param body         [Hash, String, nil]       Body of request, serialized according to `content_type`
+      # @param content_type [String, nil]             One of Mime::JSON, Mime::WWW, Mime::TEXT, Mime::MULTIPART, or `nil` for no body
+      # @param body         [Hash, Array, String, nil] Body of request, serialized according to `content_type`
+      #   For Mime::MULTIPART: Hash `name => value`, or Array of `[name, value, opts]` with opts `filename`, `content_type` for a file part
       # @param headers      [Hash]                    Headers of request
       # @return [Net::HTTPRequest] The request
       def build_request(operation, subpath, query, content_type, body, headers)
@@ -366,6 +369,9 @@ module Aspera
         when Mime::TEXT
           req.body = body
           req['Content-Type'] = Mime::TEXT
+        when Mime::MULTIPART
+          # values are kept in the request: re-sent on retry
+          req.set_form(body.to_a, Mime::MULTIPART)
         else Aspera.error_unexpected_value(content_type) { 'body type' }
         end
         # set headers
@@ -438,17 +444,18 @@ module Aspera
       end
 
       # Parse response body, according to media type
-      # @param result_http [Net::HTTPResponse] Response
-      # @param result_mime [String]            Media type of response
+      # @param http_res  [Net::HTTPResponse] Response
+      # @param mime_type [String]            Media type of response
+      # @param json_opts [Hash]              Options of `JSON.parse`
       # @return [Hash, Array, String, nil] Parsed JSON, or raw body
-      def parse_response(result_http, result_mime)
-        result_data = result_http.body
+      def parse_response(http_res, mime_type:, json_opts: {})
+        result_data = http_res.body
         Log.dump(:result_data_raw, result_data, level: :trace1)
         # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
         node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
         result_data = result_data[0..node_api_bug] if node_api_bug
-        result_mime = 'application/xml' if result_data.is_a?(String) && result_data.start_with?('<?xml')
-        result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
+        mime_type = 'application/xml' if result_data.is_a?(String) && result_data.start_with?('<?xml')
+        result_data = JSON.parse(result_data, **json_opts) if Mime.json?(mime_type) && !result_data.nil? && !result_data.empty?
         Log.dump(:result_data, result_data)
         result_data
       end

data/lib/aspera/rest/error_analyzer.rb

@@ -15,6 +15,10 @@ module Aspera
     class ErrorAnalyzer
       include Singleton
 
+      # Line of the backtrace of a Ruby exception (Exception#full_message), e.g. `\tfrom /app/x.rb:12:in 'y'`
+      BACKTRACE_LINE = /^\s+from \S.*:\d+:in /
+      private_constant :BACKTRACE_LINE
+
       attr_accessor :log_file
 
       # the singleton object is registered with application specific handlers
@@ -97,6 +101,11 @@ module Aspera
         # @param type    [String] type of exception, for logging purpose
         # @param message [String] one error message  to add to list
         def add_error(context, type, message)
+          # A server in Ruby (e.g. Orchestrator) can return the backtrace of the exception: in debug log only
+          if message.is_a?(String) && message.match?(BACKTRACE_LINE)
+            Log.log.debug { "Error with backtrace: #{message}" }
+            message = message.lines.grep_v(BACKTRACE_LINE).join.strip
+          end
           context[:messages].push(message)
           Log.log.trace1 { "Found error: #{type}: #{message}" }
           log_file = instance.log_file

data/lib/aspera/rest/list.rb

@@ -120,19 +120,19 @@ module Aspera
           Aspera.assert_values(field, ['name']) { 'Default query field' }
           query = {'q'=> value}
         end
-        lookup_entity_generic(entity: entity, field: field, value: value) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
+        lookup_entity_generic(field: field, value: value, entity: entity) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
       end
 
       # Lookup entity by field and value.
       # Extracts a single result from the list returned by the block.
       #
-      # @param entity [String] Type of entity to lookup (path, and by default it is also the field name in result)
-      # @param value  [String] Value to match against the field.
-      # @param field  [String] Field to match in the hashes (defaults to 'name').
-      # @yield []              A mandatory block that returns an Array of Hashes.
+      # @param field  [String] Field to match in the hashes.
+      # @param value  [String] Value to match against the field (exact match with `eql?`: case and type sensitive).
+      # @param entity [String] Type of entity to lookup (typically, API path, only used for exception)
+      # @yieldreturn [Array<Hash>] The list of entities to search (block is mandatory).
       # @return [Hash] The unique matching object.
       # @raise  [Cli::BadIdentifier] If 0 or >1 matches are found.
-      def lookup_entity_generic(entity:, value:, field: 'name')
+      def lookup_entity_generic(field:, value:, entity:)
         Aspera.assert(block_given?, 'block required for lookup_entity_generic')
         found = yield
         Aspera.assert_array_all(found, Hash)

data/lib/aspera/rest/util.rb

@@ -3,6 +3,7 @@
 require 'aspera/rest/parameters'
 require 'aspera/log'
 require 'aspera/assert'
+require 'aspera/secret_hider'
 require 'net/http'
 require 'net/https'
 require 'uri'
@@ -16,7 +17,12 @@ module Aspera
       # @param user [String] Username
       # @param pass [String] Password
       # @return [String] Basic auth token
-      def basic_authorization(user, pass) = "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"
+      def basic_authorization(user, pass)
+        credentials = Base64.strict_encode64("#{user}:#{pass}")
+        # Hidden wherever it appears (logs, results)
+        SecretHider.instance.add_secret_values(credentials)
+        "Basic #{credentials}"
+      end
 
       # Indicate that the given Hash query uses php style for array parameters
       # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2

data/lib/aspera/rest.rb

@@ -4,6 +4,5 @@ require 'aspera/rest/parameters'
 require 'aspera/rest/util'
 require 'aspera/rest/call_error'
 require 'aspera/rest/error_analyzer'
-require 'aspera/rest/aspera_errors'
 require 'aspera/rest/client'
 require 'aspera/rest/list'

data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml

@@ -32764,8 +32764,6 @@ components:
                 type: string
                 examples:
                   - eyJyZXNvdXJjZSI6InBhY2thZ2VzIiwidHlwZSI6ImV4dGVybmFsX3ZpZXdfcGFja2FnZSIsImlkIjoiMzAiLCJwYXNzY29kZSI6IjA2MzBkZDIzLTA3YTAtNGExYi04N2IzLWM3YjZkMGViNjE5NiIsInBhY2thZ2VfaWQiOiIzMCJ9
-                maxLength: 255
-                minLength: 1
               relays_count:
                 description: Package relays count
                 type: object