aspera-cli 4.27.2 → 4.27.4

data/lib/aspera/rest/list.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'aspera/rest/parameters'
+require 'aspera/mime'
+require 'aspera/assert'
+require 'aspera/log'
+
+module Aspera
+  module Rest
+    # List and lookup methods for Rest::Client
+    # To be included in classes inheriting Rest::Client that require those methods.
+    module List
+      # `max`: special query parameter: max number of items for list command
+      MAX_ITEMS = 'max'
+      # `pmax`: special query parameter: max number of pages for list command
+      MAX_PAGES = 'pmax'
+
+      # Query entity by general search (read with parameter `q`)
+      #
+      # @param entity [String] Path of entity in API
+      # @param value [String] Value of field of searched entity
+      # @param field [String] Field of searched entity
+      # @param query [Hash] Optional additional search query parameters
+      # @return [Hash] A single entity matching the search, or an exception if not found or multiple found
+      def lookup_with_q(entity, value:, field: 'name', query: {})
+        Aspera.assert_type(query, Hash) { 'query' }
+        Aspera.assert_type(field, String) { 'field' }
+        # returns entities matching the query (it matches against several fields in case insensitive way)
+        # We don't do paging, as anyway, we look for only one match
+        matching_items = read(entity, query.merge({'q' => value}))
+        # API style: {totalcount:, ...} cspell: disable-line: TODO: is that total_count ?
+        # @type [Array<Hash{String => String}>]
+        matching_items = matching_items[entity] if matching_items.is_a?(Hash)
+        Aspera.assert_type(matching_items, Array)
+        case matching_items.length
+        when 1 then return matching_items.first
+        when 0 then raise EntityNotFound, %Q{No such #{entity}: "#{value}"}
+        else
+          # multiple case insensitive partial matches, try case insensitive full match
+          # (AoC does not allow 2 entities with same case insensitive name, except for some, e.g. packages)
+          value_matches = matching_items.select { |i| i[field].casecmp?(value) }
+          case value_matches.length
+          when 1 then return value_matches.first
+          when 0 then raise Error, "#{entity}: Multiple case insensitive partial match for: \"#{value}\" in #{matching_items.map { |i| i[field] }.join(', ')} but no case insensitive full match. Please be more specific or give exact #{field}."
+          else raise Error, "#{entity}: Multiple entities with #{field}: \"#{value}\". Please use the identifier instead."
+          end
+        end
+      end
+
+      # Get a (full or partial) list of all entities of a given type.
+      # Using query: `offset` + `limit`
+      # And response `total_count`
+      # @param entity    [String,Symbol] API endpoint of entity to list
+      # @param items_key [String]        Key in the result to get the list of items (Default: same as `entity`)
+      # @param query     [Hash,nil]      Additional query parameters
+      # @return [Array<(Array<Hash>, Integer)>] items, total_count
+      def list_entities_limit_offset_total_count(
+        entity:,
+        operation: 'GET',
+        items_key: nil,
+        query: nil
+      )
+        entity = entity.to_s if entity.is_a?(Symbol)
+        items_key = entity.split('/').last if items_key.nil?
+        query = {} if query.nil?
+        Aspera.assert_type(entity, String)
+        Aspera.assert_type(items_key, String)
+        Aspera.assert_type(query, Hash)
+        Log.log.debug { "list_entities t=#{entity} k=#{items_key}" }
+        Log.dump(:query, query)
+        result = []
+        offset = 0
+        # Special parameters are not sent to the API (the caller's query is not modified)
+        query = query.dup
+        max_items = query.delete(MAX_ITEMS)&.to_i
+        remain_pages = query.delete(MAX_PAGES)&.to_i
+        # Merge default parameters
+        query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
+        total_count = nil
+        call_args = {
+          operation: operation,
+          subpath:   entity,
+          query:     query,
+          headers:   {'Accept' => Mime::JSON}
+        }
+        loop do
+          query['offset'] = offset
+          page_result = call(**call_args)
+          page_items = page_result[items_key]
+          Aspera.assert_type(page_items, Array)
+          result.concat(page_items)
+          # Reach the limit set by user ?
+          if !max_items.nil? && (result.length >= max_items)
+            result = result.slice(0, max_items)
+            break
+          end
+          total_count ||= page_result['total_count']
+          # An empty page ends the list, even if the total count is not reached (e.g. items deleted meanwhile)
+          break if page_items.empty?
+          # Without total count, a partial page is the last one
+          break if total_count.nil? ? page_items.length < query['limit'].to_i : result.length >= total_count
+          remain_pages -= 1 unless remain_pages.nil?
+          break if !remain_pages.nil? && remain_pages <= 0
+          offset += page_items.length
+          Parameters.instance.spinner_cb.call("#{result.length} / #{total_count || '?'}")
+        end
+        Parameters.instance.spinner_cb.call(action: :success)
+        return result, total_count
+      end
+
+      # Lookup an entity id from its name.
+      # Uses query `q` if `query` is `:default` and `field` is `name`.
+      # @param entity    [String] Type of entity to lookup, by default it is the path, and it is also the field name in result
+      # @param value     [String] Value to lookup
+      # @param field     [String] Field to match, by default it is `'name'`
+      # @param items_key [String] Key in the result to get the list of items (override entity)
+      # @param query     [Hash]   Additional query parameters (Default: `:default`)
+      def lookup_entity_by_field(entity:, value:, field: 'name', items_key: nil, query: :default)
+        if query.eql?(:default)
+          Aspera.assert_values(field, ['name']) { 'Default query field' }
+          query = {'q'=> value}
+        end
+        lookup_entity_generic(entity: entity, field: field, value: value) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
+      end
+
+      # Lookup entity by field and value.
+      # Extracts a single result from the list returned by the block.
+      #
+      # @param entity [String] Type of entity to lookup (path, and by default it is also the field name in result)
+      # @param value  [String] Value to match against the field.
+      # @param field  [String] Field to match in the hashes (defaults to 'name').
+      # @yield []              A mandatory block that returns an Array of Hashes.
+      # @return [Hash] The unique matching object.
+      # @raise  [Cli::BadIdentifier] If 0 or >1 matches are found.
+      def lookup_entity_generic(entity:, value:, field: 'name')
+        Aspera.assert(block_given?, 'block required for lookup_entity_generic')
+        found = yield
+        Aspera.assert_array_all(found, Hash)
+        found = found.select { |i| i[field].eql?(value) }
+        return found.first if found.length.eql?(1)
+        raise Cli::BadIdentifier.new(entity, value, field: field, count: found.length)
+      end
+
+      PER_PAGE_DEFAULT = 1000
+      private_constant :PER_PAGE_DEFAULT
+      module_function :lookup_entity_generic
+    end
+  end
+end

data/lib/aspera/rest/parameters.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module Aspera
+  module Rest
+    # Global settings for Rest::Client
+    # For example to remove certificate verification globally:
+    # `Parameters.instance.session_cb = lambda{|http|http.verify_mode=OpenSSL::SSL::VERIFY_NONE}`
+    #
+    # @!method self.instance
+    #   Returns the singleton instance of Parameters
+    #   @return [Parameters] the singleton instance
+    class Parameters
+      include Singleton
+
+      # @return [String] HTTP request header: `User-Agent`
+      attr_accessor :user_agent
+      # @return [String] Suffix of file being downloaded, removed when download is complete
+      attr_accessor :download_partial_suffix
+      # @return [Boolean] Retry on any error (network or HTTP)
+      attr_accessor :retry_on_error
+      # @return [Boolean] Retry on connection timeout
+      attr_accessor :retry_on_timeout
+      # @return [Boolean] Retry on HTTP code 503 (service unavailable)
+      attr_accessor :retry_on_unavailable
+      # @return [Integer] Maximum number of retries on error (first call not included)
+      attr_accessor :retry_max
+      # @return [Integer] Seconds to wait before retry
+      attr_accessor :retry_sleep
+      # @return [Proc, nil] Called on new HTTP session, with the `Net::HTTP` as argument, e.g. to set timeouts or certificate verification
+      attr_accessor :session_cb
+      # @return [Object, nil] Progress bar, receives `event` calls during download
+      attr_accessor :progress_bar
+      # @return [Proc] Called with `(title = nil, action: :spin)` to display progress of long operations (default: no display)
+      attr_accessor :spinner_cb
+
+      private
+
+      # Set default values
+      def initialize
+        @user_agent = 'RubyAsperaRest'
+        @download_partial_suffix = '.http_partial'
+        @retry_on_error = false
+        @retry_on_timeout = true
+        @retry_on_unavailable = true
+        @retry_max = 1
+        @retry_sleep = 4
+        @session_cb = nil
+        @progress_bar = nil
+        @spinner_cb = ->(_title = nil, action: :spin) {}
+      end
+    end
+  end
+end

data/lib/aspera/rest/util.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'aspera/rest/parameters'
+require 'aspera/log'
+require 'aspera/assert'
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'base64'
+
+module Aspera
+  # HTTP REST client and helpers
+  module Rest
+    class << self
+      # Build a Basic authentication header value
+      # @param user [String] Username
+      # @param pass [String] Password
+      # @return [String] Basic auth token
+      def basic_authorization(user, pass) = "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"
+
+      # Indicate that the given Hash query uses php style for array parameters
+      # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
+      # @return [Hash] The query parameters.
+      def php_style(query)
+        Aspera.assert_type(query, Hash) { 'query' }
+        query[:x_array_php_style] = true
+        query
+      end
+
+      # Build URI from URL and parameters and check it is `http` or `https`.
+      # Check if php style is specified.
+      # `nil` values in query result in key without value, e.g. `?a`, while empty string values result in `?a=`.
+      # @param url   [String]            The URL without query.
+      # @param query [Hash,Array,String] The query parameters.
+      # @return [URI] The built URI.
+      def build_uri(url, query)
+        uri = URI.parse(url)
+        Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
+        return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
+        Log.dump(:query, query)
+        uri.query =
+          case query
+          when String
+            query
+          when Hash
+            URI.encode_www_form(h_to_query_array(query))
+          when Array
+            Aspera.assert(query.all? { |i| i.is_a?(Array) && i.length.eql?(2) }, 'Query must be array of arrays of 2 elements')
+            URI.encode_www_form(query) # remove nil values
+          else Aspera.error_unexpected_value(query.class) { 'query type' }
+          end.gsub('%5B%5D=', '[]=')
+        # [] is allowed in url parameters
+        uri
+      end
+
+      # Support array for query parameter, there is no standard.
+      # Either p=1&p=2 (default)
+      # or p[]=1&p[]=2 (if `:x_array_php_style` is set to true in query)
+      # @param query [Hash] HTTP query as hash
+      # @return [Array<Array>] Array of [key, value] pairs suitable for URI.encode_www_form
+      def h_to_query_array(query)
+        Aspera.assert_type(query, Hash)
+        suffix = query[:x_array_php_style] ? '[]' : nil
+        query.each_with_object([]) do |(k, v), query_array|
+          next if k.eql?(:x_array_php_style)
+          case v
+          when Array
+            v.each do |e|
+              query_array.push(["#{k}#{suffix}", e])
+            end
+          else
+            query_array.push([k, v])
+          end
+        end
+      end
+
+      # Decode query string as Hash
+      # if parameter is only once, then it's a scalar
+      # if a parameter is several, then it's array
+      # if parameter has [] then it's an array, and [] is removed
+      # Support arrays in query string, e.g. PHP's way is p[]=1&p[]=2
+      # @param query [String] query string as in URI.query
+      # @return [Hash] decoded query
+      def query_to_h(query)
+        URI.decode_www_form(query).each_with_object({}) do |(key, value), h|
+          if key.end_with?('[]')
+            key = key[..-3]
+            h[key] = [] unless h.key?(key)
+          end
+          if h.key?(key)
+            h[key] = [h[key]] if !h[key].is_a?(Array)
+            h[key].push(value)
+          else
+            h[key] = value
+          end
+        end
+      end
+
+      # Start a HTTP/S session, also used for web sockets
+      # @param base_url [String] Base url of HTTP/S session
+      # @return [Net::HTTP] A started HTTP session
+      def start_http_session(base_url)
+        uri = URI.parse(base_url)
+        Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
+        # This honors http_proxy env var
+        http_session = Net::HTTP.new(uri.host, uri.port)
+        http_session.use_ssl = uri.scheme.eql?('https')
+        # Set http options in callback, such as timeout and cert. verification
+        Parameters.instance.session_cb&.call(http_session)
+        # Manually start session for keep alive (if supported by server, else, session is closed every time)
+        http_session.start
+        return http_session
+      end
+
+      # get Net::HTTP underlying socket i/o
+      # little hack, handy because HTTP debug, proxy, etc... will be available
+      # used implement web sockets after `start_http_session`
+      # @param http_session [Net::HTTP] the session object
+      # @return [Net::BufferedIO] The underlying socket i/o
+      def io_http_session(http_session)
+        Aspera.assert_type(http_session, Net::HTTP)
+        # Net::BufferedIO in net/protocol.rb
+        result = http_session.instance_variable_get(:@socket)
+        Aspera.assert(!result.nil?) { "no socket for #{http_session}" }
+        return result
+      end
+
+      # Get certificate chain of remote server
+      # @param url       [String]  URL of server
+      # @param as_string [Boolean] `true` to return PEM string, `false` for certificate objects
+      # @return [String, Array<OpenSSL::X509::Certificate>] Certificates of remote server
+      def remote_certificate_chain(url, as_string: true)
+        result = []
+        # initiate a session to retrieve remote certificate
+        http_session = Rest.start_http_session(url)
+        begin
+          # retrieve underlying openssl socket
+          result = Rest.io_http_session(http_session).io.peer_cert_chain
+        rescue
+          result = http_session.peer_cert
+        ensure
+          http_session.finish
+        end
+        result = result.map(&:to_pem).join("\n") if as_string
+        return result
+      end
+
+      # Parses an HTTP Content-Type header string into its media type and parameters
+      # according to RFC 9110 and RFC 6838.
+      # TODO: use gem: content_type
+      #
+      # @param header [String] The Content-Type header string, e.g., "application/json; charset=utf-8"
+      # @return [Hash] A hash with :type and :parameters keys.
+      #   Example:
+      #     {
+      #       type: "application/json",
+      #       parameters: {
+      #         charset: "utf-8",
+      #         version: "1.0"
+      #       }
+      #     }
+      def parse_header(header)
+        parts = header.split(';').map(&:strip)
+        media_type = parts.shift.downcase
+        parameters = parts.filter_map do |param|
+          key, value = param.split('=', 2)
+          next unless key && value
+          key = key.strip.downcase.to_sym
+          value = value.strip.gsub(/\A"|"\z/, '')
+          [key, value]
+        end.to_h
+        {type: media_type, parameters: parameters}
+      end
+    end
+  end
+end