aspera-cli 4.25.4 → 4.25.5

data/lib/aspera/rest.rb

@@ -72,11 +72,6 @@ module Aspera
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # Special query parameter: max number of items for list command
-    MAX_ITEMS = 'max'
-    # Special query parameter: max number of pages for list command
-    MAX_PAGES = 'pmax'
-
     class << self
       # @return [String] Basic auth token
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
@@ -174,6 +169,8 @@ module Aspera
       # get Net::HTTP underlying socket i/o
       # little hack, handy because HTTP debug, proxy, etc... will be available
       # used implement web sockets after `start_http_session`
+      # @param http_session [Net::HTTP] the session object
+      # @return [Net::BufferedIO] The underlying socket i/o
       def io_http_session(http_session)
         Aspera.assert_type(http_session, Net::HTTP)
         # Net::BufferedIO in net/protocol.rb
@@ -371,8 +368,11 @@ module Aspera
       end
       result_http = nil
       result_data = nil
+      # initialize with number of initial retries allowed, nil gives zero
+      tries_remain_redirect = @redirect_max
       # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
       begin
+        Log.log.debug("send request (retries=#{tries_remain_redirect})")
         # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
         # URI.escape()
         separator = ['', '/'].include?(subpath) ? '' : '/'
@@ -406,9 +406,6 @@ module Aspera
         Log.dump(:req_body, req.body, level: :trace1)
         # we try the call, and will retry on some error types
         error_tries ||= 1 + RestParameters.instance.retry_max
-        # initialize with number of initial retries allowed, nil gives zero
-        tries_remain_redirect = @redirect_max if tries_remain_redirect.nil?
-        Log.log.debug("send request (retries=#{tries_remain_redirect})")
         result_mime = nil
         file_saved = false
         # make http request (pipelined)
@@ -458,6 +455,9 @@ module Aspera
         # Log.log.debug{"result: body=#{result_http.body}"}
         result_data = result_http.body
         Log.dump(:result_data_raw, result_data, level: :trace1)
+        # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
+        node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
+        result_data = result_data[0..node_api_bug] if node_api_bug
         result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
         Log.dump(:result_data, result_data)
         RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
@@ -572,34 +572,6 @@ module Aspera
       return call(operation: 'CANCEL', subpath: subpath, **kwargs)
     end
 
-    # Query entity by general search (read with parameter `q`)
-    # TODO: not generic enough ? move somewhere ? inheritance ?
-    # @param subpath     [String] Path of entity in API
-    # @param search_name [String] Name of searched entity
-    # @param query       [Hash]   Additional search query parameters
-    # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
-    def lookup_by_name(subpath, search_name, query: nil)
-      query = {} if query.nil?
-      # returns entities matching the query (it matches against several fields in case insensitive way)
-      matching_items = read(subpath, query.merge({'q' => search_name}))
-      # API style: {totalcount:, ...} cspell: disable-line
-      matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
-      Aspera.assert_type(matching_items, Array)
-      case matching_items.length
-      when 1 then return matching_items.first
-      when 0 then raise EntityNotFound, %Q{No such #{subpath}: "#{search_name}"}
-      else
-        # multiple case insensitive partial matches, try case insensitive full match
-        # (anyway AoC does not allow creation of 2 entities with same case insensitive name)
-        name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
-        case name_matches.length
-        when 1 then return name_matches.first
-        when 0 then raise %Q(#{subpath}: Multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
-        else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
-        end
-      end
-    end
-
     UNAVAILABLE_CODES = ['503']
 
     private_constant :UNAVAILABLE_CODES

data/lib/aspera/rest_list.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Aspera
+  # List and lookup methods for Rest
+  # To be included in classes inheriting Rest that require those methods.
+  module RestList
+    # `max`: special query parameter: max number of items for list command
+    MAX_ITEMS = 'max'
+    # `pmax`: special query parameter: max number of pages for list command
+    MAX_PAGES = 'pmax'
+
+    # Query entity by general search (read with parameter `q`)
+    #
+    # @param subpath     [String] Path of entity in API
+    # @param search_name [String] Name of searched entity
+    # @param query       [Hash]   Optional additional search query parameters
+    # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
+    def lookup_by_name(subpath, search_name, query: nil)
+      query = {} if query.nil?
+      # returns entities matching the query (it matches against several fields in case insensitive way)
+      matching_items = read(subpath, query.merge({'q' => search_name}))
+      # API style: {totalcount:, ...} cspell: disable-line
+      matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
+      Aspera.assert_type(matching_items, Array)
+      case matching_items.length
+      when 1 then return matching_items.first
+      when 0 then raise EntityNotFound, %Q{No such #{subpath}: "#{search_name}"}
+      else
+        # multiple case insensitive partial matches, try case insensitive full match
+        # (anyway AoC does not allow creation of 2 entities with same case insensitive name)
+        name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
+        case name_matches.length
+        when 1 then return name_matches.first
+        when 0 then raise %Q(#{subpath}: Multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
+        else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
+        end
+      end
+    end
+
+    # Get a (full or partial) list of all entities of a given type with query: offset/limit
+    # @param entity    [String,Symbol] API endpoint of entity to list
+    # @param items_key [String]        Key in the result to get the list of items (Default: same as `entity`)
+    # @param query     [Hash,nil]      Additional query parameters
+    # @return [Array<(Array<Hash>, Integer)>] items, total_count
+    def list_entities_limit_offset_total_count(
+      entity:,
+      items_key: nil,
+      query: nil
+    )
+      entity = entity.to_s if entity.is_a?(Symbol)
+      items_key = entity.split('/').last if items_key.nil?
+      query = {} if query.nil?
+      Aspera.assert_type(entity, String)
+      Aspera.assert_type(items_key, String)
+      Aspera.assert_type(query, Hash)
+      Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
+      result = []
+      offset = 0
+      max_items = query.delete(MAX_ITEMS)
+      remain_pages = query.delete(MAX_PAGES)
+      # Merge default parameters, by default 100 per page
+      query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
+      total_count = nil
+      loop do
+        query['offset'] = offset
+        page_result = read(entity, query)
+        Aspera.assert_type(page_result[items_key], Array)
+        result.concat(page_result[items_key])
+        # Reach the limit set by user ?
+        if !max_items.nil? && (result.length >= max_items)
+          result = result.slice(0, max_items)
+          break
+        end
+        total_count ||= page_result['total_count']
+        break if result.length >= total_count
+        remain_pages -= 1 unless remain_pages.nil?
+        break if remain_pages == 0
+        offset += page_result[items_key].length
+        RestParameters.instance.spinner_cb.call("#{result.length} / #{total_count || '?'}")
+      end
+      RestParameters.instance.spinner_cb.call(action: :success)
+      return result, total_count
+    end
+
+    # Lookup an entity id from its name.
+    # Uses query `q` if `query` is `:default` and `field` is `name`.
+    # @param entity    [String] Type of entity to lookup, by default it is the path, and it is also the field name in result
+    # @param value     [String] Value to lookup
+    # @param field     [String] Field to match, by default it is `'name'`
+    # @param items_key [String] Key in the result to get the list of items (override entity)
+    # @param query     [Hash]   Additional query parameters (Default: `:default`)
+    def lookup_entity_by_field(entity:, value:, field: 'name', items_key: nil, query: :default)
+      if query.eql?(:default)
+        Aspera.assert(field.eql?('name')){'Default query is on name only'}
+        query = {'q'=> value}
+      end
+      lookup_entity_generic(entity: entity, field: field, value: value){list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first}
+    end
+
+    # Lookup entity by field and value.
+    # Extracts a single result from the list returned by the block.
+    #
+    # @param entity [String] Type of entity to lookup (path, and by default it is also the field name in result)
+    # @param value  [String] Value to match against the field.
+    # @param field  [String] Field to match in the hashes (defaults to 'name').
+    # @yield []              A mandatory block that returns an Array of Hashes.
+    # @return [Hash] The unique matching object.
+    # @raise  [Cli::BadIdentifier] If 0 or >1 matches are found.
+    def lookup_entity_generic(entity:, value:, field: 'name')
+      Aspera.assert(block_given?)
+      found = yield
+      Aspera.assert_array_all(found, Hash)
+      found = found.select{ |i| i[field].eql?(value)}
+      return found.first if found.length.eql?(1)
+      raise Cli::BadIdentifier.new(entity, value, field: field, count: found.length)
+    end
+    PER_PAGE_DEFAULT = 1000
+    private_constant :PER_PAGE_DEFAULT
+    module_function :lookup_entity_generic
+  end
+end

data/lib/aspera/sync/operations.rb

@@ -342,7 +342,7 @@ module Aspera
       CONF_SCHEMA = CommandLineBuilder.read_schema(__dir__, 'conf')
       CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
       ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
-      PRIVATE_FOLDER = '.private-asp'
+      PRIVATE_FOLDER = "#{Environment.instance.os.eql?(Environment::OS_WINDOWS) ? '~' : '.'}private-asp"
       ASYNC_DB = 'snap.db'
       PARAM_KEYS = %w[local sessions].freeze
 

data/lib/aspera/transfer/parameters.rb

@@ -38,19 +38,19 @@ module Aspera
           @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
         end
 
-        # File list is provided directly with ascp arguments
-        # @columns ascp_args [Array,NilClass] ascp arguments
+        # File list is provided directly with `ascp` arguments
+        # @columns ascp_args [Array,NilClass] `ascp` arguments
         def ascp_args_file_list?(ascp_args)
           ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
         end
       end
 
-      # @param job_spec        [Hash]    Transfer spec
-      # @param ascp_args       [Array]   Other ascp args
-      # @param quiet           [Boolean] Remove ascp output
-      # @param trusted_certs   [Array]   Trusted certificates
-      # @param client_ssh_key  [:rsa,:dsa] :rsa or :dsa
-      # @param check_ignore_cb [Proc]    Callback
+      # @param job_spec        [Hash]          Transfer specification
+      # @param ascp_args       [Array]         Other `ascp` args
+      # @param quiet           [Boolean]       Remove `ascp` progress bar if `true`
+      # @param client_ssh_key  [:rsa,:dsa]     Type of Aspera Client SSH key to use.
+      # @param trusted_certs   [Array<String>] List of path to trusted certificates stores when using WSS
+      # @param check_ignore_cb [Proc]          Callback to check if WSS connection shall ignore certificate validity
       def initialize(
         job_spec,
         ascp_args:       nil,

data/lib/aspera/yaml.rb

@@ -25,7 +25,7 @@ module Aspera
         counts.each do |key_str, count|
           next if count <= 1
           path = (parent_path + [key_str]).join('.')
-          occurrences = key_nodes[key_str].map{ |kn| kn.start_line ? kn.start_line + 1 : 'unknown'}.map(&:to_s).join(', ')
+          occurrences = key_nodes[key_str].map{ |kn| kn.start_line ? kn.start_line + 1 : 'unknown'}.join(', ')
           duplicate_keys << "#{path}: #{occurrences}"
         end
       else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.25.4
+  version: 4.25.5
 platform: ruby
 authors:
 - Laurent Martin
@@ -424,6 +424,7 @@ files:
 - lib/aspera/rest_call_error.rb
 - lib/aspera/rest_error_analyzer.rb
 - lib/aspera/rest_errors_aspera.rb
+- lib/aspera/rest_list.rb
 - lib/aspera/secret_hider.rb
 - lib/aspera/ssh.rb
 - lib/aspera/ssl.rb
@@ -456,7 +457,7 @@ metadata:
   source_code_uri: https://github.com/IBM/aspera-cli/tree/main/lib/aspera
   changelog_uri: https://github.com/IBM/aspera-cli/blob/main/CHANGELOG.md
   rubygems_uri: https://rubygems.org/gems/aspera-cli
-  documentation_uri: https://www.rubydoc.info/gems/aspera-cli
+  documentation_uri: https://ibm.biz/ascli-doc
 rdoc_options: []
 require_paths:
 - lib
@@ -472,7 +473,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - Read the manual for any requirement
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: 'Execute actions using command line on IBM Aspera Server products: Aspera
   on Cloud, Faspex, Shares, Node, Console, Orchestrator, High Speed Transfer Server'