aspera-cli 4.24.0 → 4.24.2

data/lib/aspera/rest.rb

@@ -54,80 +54,91 @@ module Aspera
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # flag for array parameters prefixed with []
-    ARRAY_PARAMS = '[]'
-
-    private_constant :ARRAY_PARAMS
-
-    # error message when entity not found (TODO: use specific exception)
+    # Error message when entity not found (TODO: use specific exception)
     ENTITY_NOT_FOUND = 'No such'
 
     MIME_JSON = 'application/json'
     MIME_WWW = 'application/x-www-form-urlencoded'
     MIME_TEXT = 'text/plain'
 
-    # Content-Type that are JSON
-    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
-
-    UNAVAILABLE_CODES = ['503']
+    # Special query parameter: max number of items for list command
+    MAX_ITEMS = 'max'
+    # Special query parameter: max number of pages for list command
+    MAX_PAGES = 'pmax'
 
     class << self
       # @return [String] Basic auth token
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
 
-      # Build a parameter list prefixed with "[]"
-      # @param values [Array] list of values
-      def array_params(values)
-        return [ARRAY_PARAMS].concat(values)
-      end
-
-      def array_params?(values)
-        return values.first.eql?(ARRAY_PARAMS)
+      # Indicate that the given Hash query uses php style for array parameters
+      # a[]=1&a[]=2
+      def php_style(query)
+        Aspera.assert_type(query, Hash){'query'}
+        query[:x_array_php_style] = true
+        query
       end
 
       # Build URI from URL and parameters and check it is http or https
-      # encode array [] parameters
-      # @param query [Hash,Array]
-      def build_uri(url, query = nil)
+      # Check iof php style is specified
+      # @param url   [String]            The URL without query.
+      # @param query [Hash,Array,String] The query.
+      def build_uri(url, query)
         uri = URI.parse(url)
-        Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
+        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
         return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
         Log.dump(:query, query)
-        query_array = []
-        case query
-        when Hash
-          query.each do |k, v|
-            case v
-            when Array
-              # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
-              suffix = array_params?(v) ? v.shift : ''
-              v.each do |e|
-                query_array.push(["#{k}#{suffix}", e])
-              end
-            else
-              query_array.push([k, v])
+        uri.query =
+          case query
+          when String
+            query
+          when Hash
+            URI.encode_www_form(h_to_query_array(query))
+          when Array
+            Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays of 2 elements'}
+            URI.encode_www_form(query)
+          else Aspera.error_unexpected_value(query.class){'query type'}
+          end.gsub('%5B%5D=', '[]=')
+        # [] is allowed in url parameters
+        uri
+      end
+
+      # @param query [Hash] HTTP query as hash
+      def h_to_query_array(query)
+        Aspera.assert_type(query, Hash)
+        # Support array for query parameter, there is no standard.
+        # Either p[]=1&p[]=2, or p=1&p=2
+        suffix = query.delete(:x_array_php_style) ? '[]' : nil
+        query.each_with_object([]) do |(k, v), query_array|
+          case v
+          when Array
+            v.each do |e|
+              query_array.push(["#{k}#{suffix}", e])
             end
+          else
+            query_array.push([k, v])
           end
-        when Array
-          Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays or 2 elements'}
-          query_array = query
-        else
-          raise "Query must be Hash or Array, not #{query.class}"
         end
-        # [] is allowed in url parameters
-        uri.query = URI.encode_www_form(query_array).gsub('%5B%5D=', '[]=')
-        return uri
       end
 
       # Decode query string as Hash
-      # Does not support arrays in query string, no standard, e.g. PHP's way is p[]=1&p[]=2
+      # if parameter is only once, then it's a scalar
+      # if a parameter is several, then it's array
+      # if parameter has [] then it's an array, and [] is removed
+      # Support arrays in query string, e.g. PHP's way is p[]=1&p[]=2
       # @param query [String] query string as in URI.query
       # @return [Hash] decoded query
       def query_to_h(query)
-        URI.decode_www_form(query).each_with_object({}) do |pair, h|
-          key = pair.first
-          raise "Array not supported in query string: #{key}" if key.include?('[]') || h.key?(key)
-          h[key] = pair.last
+        URI.decode_www_form(query).each_with_object({}) do |(key, value), h|
+          if key.end_with?('[]')
+            key = key[..-3]
+            h[key] = [] unless h.key?(key)
+          end
+          if h.key?(key)
+            h[key] = [h[key]] if !h[key].is_a?(Array)
+            h[key].push(value)
+          else
+            h[key] = value
+          end
         end
       end
 
@@ -135,13 +146,14 @@ module Aspera
       # @param base_url [String] base url of HTTP/S session
       # @return [Net::HTTP] a started HTTP session
       def start_http_session(base_url)
-        uri = build_uri(base_url)
-        # this honors http_proxy env var
+        uri = URI.parse(base_url)
+        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
+        # This honors http_proxy env var
         http_session = Net::HTTP.new(uri.host, uri.port)
         http_session.use_ssl = uri.scheme.eql?('https')
-        # set http options in callback, such as timeout and cert. verification
+        # Set http options in callback, such as timeout and cert. verification
         RestParameters.instance.session_cb&.call(http_session)
-        # manually start session for keep alive (if supported by server, else, session is closed every time)
+        # Manually start session for keep alive (if supported by server, else, session is closed every time)
         http_session.start
         return http_session
       end
@@ -174,19 +186,31 @@ module Aspera
         return result
       end
 
-      # Parse a header string as returned by HTTP
-      # @param header [String] header string, e.g. "application/json; charset=utf-8"
-      # @return [Hash] parsed header with type and parameters
-      #   {type: 'application/json', parameters: {charset: 'utf-8'}}
+      # Parses an HTTP Content-Type header string into its media type and parameters
+      # according to RFC 9110 and RFC 6838.
+      # TODO: use gem: content_type
+      #
+      # @param header [String] The Content-Type header string, e.g., "application/json; charset=utf-8"
+      # @return [Hash] A hash with :type and :parameters keys.
+      #   Example:
+      #     {
+      #       type: "application/json",
+      #       parameters: {
+      #         charset: "utf-8",
+      #         version: "1.0"
+      #       }
+      #     }
       def parse_header(header)
-        type, *params = header.split(/;\s*/)
-        parameters = params.map do |param|
-          one = param.split(/=\s*/)
-          one[0] = one[0].to_sym
-          one[1] = one[1].gsub(/\A"|"\z/, '')
-          one
+        parts = header.split(';').map(&:strip)
+        media_type = parts.shift.downcase
+        parameters = parts.filter_map do |param|
+          key, value = param.split('=', 2)
+          next unless key && value
+          key = key.strip.downcase.to_sym
+          value = value.strip.gsub(/\A"|"\z/, '')
+          [key, value]
         end.to_h
-        {type: type.downcase, parameters: parameters}
+        {type: media_type, parameters: parameters}
       end
     end
 
@@ -200,8 +224,13 @@ module Aspera
 
     public
 
-    attr_reader :base_url
+    # All original constructor parameters
     attr_reader :auth_params
+
+    # The root URL for the API
+    attr_reader :base_url
+
+    # Base common headers of API
     attr_reader :headers
 
     # @return creation parameters
@@ -278,7 +307,7 @@ module Aspera
     # @param body [Hash, String] body parameters
     # @param headers [Hash] additional headers (override Content-Type)
     # @param save_to_file (filepath)
-    # @param return_error (bool)
+    # @param exception (bool) true, error raise exception
     def call(
       operation:,
       subpath: nil,
@@ -287,12 +316,14 @@ module Aspera
       body: nil,
       headers: nil,
       save_to_file: nil,
-      return_error: false
+      exception: true
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
       Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
-      Log.dump(:body, body)
+      Log.dump(:body, body, level: :trace1)
+      Log.dump(:query, query, level: :trace1)
+      Log.dump(:headers, headers, level: :trace1)
       Aspera.assert_type(subpath, String)
       if headers.nil?
         headers = @headers.clone
@@ -462,12 +493,12 @@ module Aspera
             body: body,
             content_type: content_type,
             save_to_file: save_to_file,
-            return_error: return_error,
+            exception: exception,
             headers: headers
           )
         end
         # raise exception if could not retry and not return error in result
-        raise e unless return_error
+        raise e if exception
       end
       Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
       return result
@@ -478,31 +509,36 @@ module Aspera
     # If specific elements are needed, then use the full `call` method
     #
 
-    def create(subpath, params)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
+    # Create: `POST`
+    def create(subpath, params, **kwargs)
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
     end
 
-    def read(subpath, query = nil)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
+    # Read: `GET`
+    def read(subpath, query = nil, **kwargs)
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)[:data]
     end
 
-    def update(subpath, params)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
+    # Update: `PUT`
+    def update(subpath, params, **kwargs)
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
     end
 
-    def delete(subpath, params = nil)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
+    # Delete: `DELETE`
+    def delete(subpath, params = nil, **kwargs)
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)[:data]
     end
 
-    def cancel(subpath)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON})[:data]
+    # Cancel: `CANCEL`
+    def cancel(subpath, **kwargs)
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)[:data]
     end
 
     # Query entity by general search (read with parameter `q`)
     # TODO: not generic enough ? move somewhere ? inheritance ?
-    # @param subpath path of entity in API
-    # @param search_name name of searched entity
-    # @param query additional search query parameters
+    # @param subpath     [String] Path of entity in API
+    # @param search_name [String] Name of searched entity
+    # @param query       [Hash]   Additional search query parameters
     # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
     def lookup_by_name(subpath, search_name, query: nil)
       query = {} if query.nil?
@@ -520,10 +556,17 @@ module Aspera
         name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
         case name_matches.length
         when 1 then return name_matches.first
-        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
+        when 0 then raise %Q(#{subpath}: Multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
         else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
         end
       end
     end
+
+    # Content-Type that are JSON
+    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
+
+    UNAVAILABLE_CODES = ['503']
+
+    private_constant :JSON_DECODE, :UNAVAILABLE_CODES
   end
 end

data/lib/aspera/ssh.rb

@@ -17,7 +17,7 @@ module Aspera
         $VERBOSE = nil
         Net::SSH::Authentication::Session.class_eval do
           define_method(:default_keys) do
-            %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa].freeze
+            %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
           end
           private(:default_keys)
         end rescue nil
@@ -81,6 +81,6 @@ module Aspera
   end
 end
 
-# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
-Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
+# Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
+Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none?
 Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')

data/lib/aspera/sync/args.schema.yaml

@@ -1,22 +1,30 @@
 $schema: https://json-schema.org/draft/2020-12/schema
 $id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/transfer/sync_instance.schema.yaml
+title: Aspera Sync job description (args)
+description: Instance (global) and Session (per-sync) parameters for async, specific to `ascli`.
 $comment: >-
-  `x-` fields documented in `command_line_builder.rb`
   This spec is specific to `ascli`.
   The native async `conf` format is now preferred.
-title: SyncInstanceSpec
-description: Instance (global) and Session (per-sync) parameters for async.
+  `x-` fields are documented in `command_line_builder.rb`
 type: object
 properties:
   alt_logdir:
     type: string
+    x-cli-option: true
   watchd:
     type: string
+    x-cli-option: true
   apply_local_docroot:
+    type: boolean
+    x-cli-option: true
     x-cli-switch: true
   quiet:
+    type: boolean
+    x-cli-option: true
     x-cli-switch: true
   ws_connect:
+    type: boolean
+    x-cli-option: true
     x-cli-switch: true
   sessions:
     type: array
@@ -26,27 +34,36 @@ properties:
       properties:
         name:
           type: string
+          x-cli-option: true
         local_dir:
           type: string
+          x-cli-option: true
         remote_dir:
           type: string
+          x-cli-option: true
         local_db_dir:
           type: string
+          x-cli-option: true
         remote_db_dir:
           type: string
+          x-cli-option: true
         host:
           type: string
+          x-cli-option: true
           x-ts-name: remote_host
         user:
           type: string
+          x-cli-option: true
           x-ts-name: remote_user
         private_key_paths:
           type: array
           x-cli-option: "--private-key-path"
         direction:
           type: string
+          x-cli-option: true
         checksum:
           type: string
+          x-cli-option: true
         tags:
           type: object
           x-cli-option: "--tags64"
@@ -54,49 +71,75 @@ properties:
           x-ts-name: tags
         tcp_port:
           type: integer
+          x-cli-option: true
           x-ts-name: ssh_port
         rate_policy:
           type: string
+          x-cli-option: true
         target_rate:
           type: string
+          x-cli-option: true
         cooloff:
           type: integer
+          x-cli-option: true
         pending_max:
           type: integer
+          x-cli-option: true
         scan_intensity:
           type: string
+          x-cli-option: true
         cipher:
           type: string
+          x-cli-option: true
           x-cli-convert: remove_hyphen
           x-ts-name: cipher
         transfer_threads:
           type: integer
+          x-cli-option: true
         preserve_time:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
           x-ts-name: preserve_times
         preserve_access_time:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
         preserve_modification_time:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
         preserve_uid:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
           x-ts-name: preserve_file_owner_uid
         preserve_gid:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
           x-ts-name: preserve_file_owner_gid
         create_dir:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
           x-ts-name: create_dir
         reset:
+          type: boolean
+          x-cli-option: true
           x-cli-switch: true
         remote_password:
+          type: string
           x-cli-envvar: ASPERA_SCP_PASS
           x-ts-name: remote_password
         cookie:
+          type: string
           x-cli-envvar: ASPERA_SCP_COOKIE
           x-ts-name: cookie
         token:
+          type: string
           x-cli-envvar: ASPERA_SCP_TOKEN
           x-ts-name: token
         license:
+          type: string
           x-cli-envvar: ASPERA_SCP_LICENSE