aspera-cli 4.24.1 → 4.24.2

data/lib/aspera/transfer/spec_doc.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'aspera/agent/base'
+require 'aspera/agent/factory'
 
 module Aspera
   module Transfer
@@ -13,33 +13,38 @@ module Aspera
         end
 
         # @param formatter      [Cli::Formatter] Formatter to use, methods: special_format, check_row
-        # @param include_option [Boolean] true : include CLI options
+        # @param include_option [Boolean]        `true` : include CLI options
+        # @param agent_columns  [Boolean]        `true` : include agents columns
+        # @param schema         [Hash]           The JSON spec
         # @return [Array] a table suitable to display in manual
-        def man_table(formatter, include_option: false, agent_columns: true)
+        def man_table(formatter, include_option: false, agent_columns: true, schema: Spec::SCHEMA)
           col_local = agent_to_short(:direct)
-          cols = agent_columns ? %i[name type] + AGENT_LIST.map(&:last) + %i[description] : %i[name type description]
-          rows = Spec::SCHEMA['properties'].filter_map do |name, properties|
+          cols = %i[name type description]
+          cols.insert(-2, *AGENT_LIST.map(&:last)) if agent_columns
+          rows = []
+          schema['properties'].each do |name, info|
+            if info['type'].eql?('object') && info['properties']
+              rows.concat(man_table(formatter, include_option: include_option, agent_columns: agent_columns, schema: info).last.map { |h| h.merge(name: "#{name}.#{h[:name]}") })
+            end
             # manual table
             columns = {
               name:        name,
-              type:        properties['type'],
+              type:        info['type'],
               description: []
             }
-            # replace "back solidus" HTML entity with its text value
-            #
-            # split lines
+            # replace "back solidus" HTML entity with its text value, highlight keywords, and split lines
             columns[:description] =
-              properties['description']
+              info['description']
                 .gsub('\', '\\')
                 .gsub(/`([a-z0-9_.+-]+)`/){formatter.keyword_highlight(Regexp.last_match(1))}
-                .split("\n") if properties.key?('description')
-            columns[:description].unshift("DEPRECATED: #{properties['x-deprecation']}") if properties.key?('x-deprecation')
+                .split("\n") if info.key?('description')
+            columns[:description].unshift("DEPRECATED: #{info['x-deprecation']}") if info.key?('x-deprecation')
             # add flags for supported agents in doc
             agents = []
             AGENT_LIST.each do |agent_info|
-              agents.push(agent_info.last) if properties['x-agents'].nil? || properties['x-agents'].include?(agent_info.first.to_s)
+              agents.push(agent_info.last) if info['x-agents'].nil? || info['x-agents'].include?(agent_info.first.to_s)
             end
-            agents.push(col_local) if properties['x-cli-option'] && !agents.include?(col_local)
+            Aspera.assert(agents.include?(col_local)){"#{name}: x-cli-option requires agent direct (or nil)"} if info['x-cli-option']
             if agent_columns
               AGENT_LIST.each do |agent_info|
                 columns[agent_info.last] = formatter.tick(agents.include?(agent_info.last))
@@ -49,30 +54,31 @@ module Aspera
             end
             # only keep lines that are usable in supported agents
             next false if agents.empty?
-            columns[:description].push("Allowed values: #{properties['enum'].map{ |v| formatter.keyword_highlight(v)}.join(', ')}") if properties.key?('enum')
-            envvar = ''
-            cli_option =
-              if properties.key?('x-cli-envvar')
-                envvar = 'env:'
-                properties['x-cli-envvar']
-              elsif properties['x-cli-switch']
-                properties['x-cli-option']
-              elsif properties['x-cli-special']
-                ''
-              elsif properties['x-cli-option']
-                arg_type = properties.key?('enum') ? '{enum}' : "{#{[properties['type']].flatten.join('|')}}"
-                conversion_tag = properties.key?('x-cli-convert') ? '(conversion)' : ''
-                sep = properties['x-cli-option'].start_with?('--') ? '=' : ' '
-                "#{properties['x-cli-option']}#{sep}#{conversion_tag}#{arg_type}"
-              end
-            columns[:description].push("(#{envvar}#{formatter.keyword_highlight(cli_option)})") if include_option && !cli_option.to_s.empty?
-            formatter.check_row(columns)
-          end.sort_by{ |i| i[:name]}
-          [cols, rows]
+            columns[:description].push("Allowed values: #{info['enum'].map{ |v| formatter.keyword_highlight(v)}.join(', ')}") if info.key?('enum')
+            if include_option
+              envvar_prefix = ''
+              cli_option =
+                if info.key?('x-cli-envvar')
+                  envvar_prefix = 'env:'
+                  info['x-cli-envvar']
+                elsif info['x-cli-switch']
+                  info['x-cli-option']
+                elsif info['x-cli-option']
+                  arg_type = info.key?('enum') ? '{enum}' : "{#{[info['type']].flatten.join('|')}}"
+                  # conversion_tag = info['x-cli-convert']
+                  conversion_tag = info.key?('x-cli-convert') ? 'conversion' : nil
+                  sep = info['x-cli-option'].start_with?('--') ? '=' : ' '
+                  "#{info['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
+                end
+              columns[:description].push("(#{'special:' if info['x-cli-special']}#{envvar_prefix}#{formatter.keyword_highlight(cli_option)})") if cli_option
+            end
+            rows.push(formatter.check_row(columns))
+          end
+          [cols, rows.sort_by{ |i| i[:name]}]
         end
       end
       # Agents shown in manual for parameters (sub list)
-      AGENT_LIST = Agent::Base.agent_list.map do |agent_sym|
+      AGENT_LIST = Agent::Factory.instance.list.map do |agent_sym|
         [agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
       end.sort_by(&:last).freeze
     end

data/lib/aspera/uri_reader.rb

@@ -37,7 +37,7 @@ module Aspera
           return File.expand_path(url[SCHEME_FILE_PFX2.length..-1])
         else
           # download to temp file
-          # autodelete on exit
+          # auto-delete on exit
           sdk_archive_path = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
           Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
           return sdk_archive_path

data/lib/aspera/web_auth.rb

@@ -17,7 +17,7 @@ module Aspera
     def service(request, response)
       Log.log.debug{"received request from browser #{request.request_method} #{request.path}"}
       Aspera.assert_values(request.request_method, ['GET'], type: WEBrick::HTTPStatus::MethodNotAllowed){'HTTP verb'}
-      additionnal_info = @web_auth.signal_request(request)
+      additional_info = @web_auth.signal_request(request)
       response.status = 200
       response.content_type = 'text/html'
       response.body = <<~HTML
@@ -91,7 +91,7 @@ module Aspera
         <body>
         <h1>Thank You!</h1>
         <p>You can close this window.</p>
-        <p>#{additionnal_info}</p>
+        <p>#{additional_info}</p>
 
         <!-- JavaScript to generate IBM logos -->
         <script>
@@ -170,15 +170,15 @@ module Aspera
   # store the final query
   class WebAuth < WebServerSimple
     # @param endpoint_url     [String] e.g. 'https://127.0.0.1:12345'
-    # @param additionnal_info [String] Information in web page
-    def initialize(endpoint_url, additionnal_info = nil)
+    # @param additional_info [String] Information in web page
+    def initialize(endpoint_url, additional_info = nil)
       uri = URI.parse(endpoint_url)
       super(uri)
       @mutex = Mutex.new
       @cond = ConditionVariable.new
       @expected_path = uri.path.empty? ? '/' : uri.path
       @query = nil
-      @additionnal_info = additionnal_info
+      @additional_info = additional_info
       # last argument (self) is provided to constructor of servlet
       mount(@expected_path, WebAuthServlet, self)
       # server runs in thread
@@ -194,7 +194,7 @@ module Aspera
         @query = request.query
         @cond.signal
       end
-      return @additionnal_info
+      return @additional_info
     end
 
     # wait for request on web server (main thread)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.24.1
+  version: 4.24.2
 platform: ruby
 authors:
 - Laurent Martin
@@ -37,7 +37,7 @@ cert_chain:
   eTf9kxhVM40wGQOECVNA8UsEEZHD48eF+csUYZtAJOF5oxTI8UyV9T/o6CgO0c9/
   Gzz+Qm5ULOUcPiJLjSpaiTrkiIVYiDGnqNSr6R1Hb1c=
   -----END CERTIFICATE-----
-date: 2025-10-02 00:00:00.000000000 Z
+date: 2025-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: blankslate
@@ -282,6 +282,7 @@ files:
 - lib/aspera/agent/connect.rb
 - lib/aspera/agent/desktop.rb
 - lib/aspera/agent/direct.rb
+- lib/aspera/agent/factory.rb
 - lib/aspera/agent/httpgw.rb
 - lib/aspera/agent/node.rb
 - lib/aspera/agent/transferd.rb
@@ -289,13 +290,13 @@ files:
 - lib/aspera/api/aoc.rb
 - lib/aspera/api/ats.rb
 - lib/aspera/api/cos_node.rb
+- lib/aspera/api/faspex.rb
 - lib/aspera/api/httpgw.rb
 - lib/aspera/api/node.rb
 - lib/aspera/ascmd.rb
 - lib/aspera/ascp/installation.rb
 - lib/aspera/ascp/management.rb
 - lib/aspera/assert.rb
-- lib/aspera/cli/basic_auth_plugin.rb
 - lib/aspera/cli/error.rb
 - lib/aspera/cli/extended_value.rb
 - lib/aspera/cli/formatter.rb
@@ -303,19 +304,21 @@ files:
 - lib/aspera/cli/info.rb
 - lib/aspera/cli/main.rb
 - lib/aspera/cli/manager.rb
-- lib/aspera/cli/plugin.rb
-- lib/aspera/cli/plugin_factory.rb
 - lib/aspera/cli/plugins/alee.rb
 - lib/aspera/cli/plugins/aoc.rb
 - lib/aspera/cli/plugins/ats.rb
+- lib/aspera/cli/plugins/base.rb
+- lib/aspera/cli/plugins/basic_auth.rb
 - lib/aspera/cli/plugins/config.rb
 - lib/aspera/cli/plugins/console.rb
 - lib/aspera/cli/plugins/cos.rb
+- lib/aspera/cli/plugins/factory.rb
 - lib/aspera/cli/plugins/faspex.rb
 - lib/aspera/cli/plugins/faspex5.rb
 - lib/aspera/cli/plugins/faspio.rb
 - lib/aspera/cli/plugins/httpgw.rb
 - lib/aspera/cli/plugins/node.rb
+- lib/aspera/cli/plugins/oauth.rb
 - lib/aspera/cli/plugins/orchestrator.rb
 - lib/aspera/cli/plugins/preview.rb
 - lib/aspera/cli/plugins/server.rb
@@ -378,7 +381,6 @@ files:
 - lib/aspera/rest_call_error.rb
 - lib/aspera/rest_error_analyzer.rb
 - lib/aspera/rest_errors_aspera.rb
-- lib/aspera/resumer.rb
 - lib/aspera/secret_hider.rb
 - lib/aspera/ssh.rb
 - lib/aspera/sync/args.schema.yaml
@@ -388,9 +390,9 @@ files:
 - lib/aspera/temp_file_manager.rb
 - lib/aspera/timer_limiter.rb
 - lib/aspera/transfer/error.rb
-- lib/aspera/transfer/error_info.rb
 - lib/aspera/transfer/faux_file.rb
 - lib/aspera/transfer/parameters.rb
+- lib/aspera/transfer/resumer.rb
 - lib/aspera/transfer/spec.rb
 - lib/aspera/transfer/spec.schema.yaml
 - lib/aspera/transfer/spec_doc.rb

data/lib/aspera/cli/basic_auth_plugin.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-require 'aspera/rest'
-require 'aspera/cli/plugin'
-
-module Aspera
-  module Cli
-    # base class for applications supporting basic authentication
-    class BasicAuthPlugin < Cli::Plugin
-      class << self
-        def declare_options(options)
-          options.declare(:url, 'URL of application, e.g. https://app.example.com/aspera/app')
-          options.declare(:username, "User's identifier")
-          options.declare(:password, "User's password")
-          options.parse_options!
-        end
-      end
-
-      def initialize(context:, basic_options: true)
-        super(context: context)
-        BasicAuthPlugin.declare_options(options) if basic_options
-      end
-
-      # returns a Rest object with basic auth
-      def basic_auth_params(subpath = nil)
-        api_url = options.get_option(:url, mandatory: true)
-        api_url = "#{api_url}/#{subpath}" unless subpath.nil?
-        return {
-          base_url: api_url,
-          auth:     {
-            type:     :basic,
-            username: options.get_option(:username, mandatory: true),
-            password: options.get_option(:password, mandatory: true)
-          }
-        }
-      end
-
-      def basic_auth_api(subpath = nil)
-        return Rest.new(**basic_auth_params(subpath))
-      end
-    end
-  end
-end

data/lib/aspera/cli/plugin.rb

@@ -1,333 +0,0 @@
-# frozen_string_literal: true
-
-require 'aspera/cli/extended_value'
-require 'aspera/assert'
-
-module Aspera
-  module Cli
-    # Base class for plugins
-    class Plugin
-      # operations without id
-      GLOBAL_OPS = %i[create list].freeze
-      # operations with id
-      INSTANCE_OPS = %i[modify delete show].freeze
-      # all standard operations
-      ALL_OPS = (GLOBAL_OPS + INSTANCE_OPS).freeze
-      # special query parameter: max number of items for list command
-      MAX_ITEMS = 'max'
-      # special query parameter: max number of pages for list command
-      MAX_PAGES = 'pmax'
-      # special identifier format: look for this name to find where supported
-      REGEX_LOOKUP_ID_BY_FIELD = /^%([^:]+):(.*)$/
-      PER_PAGE_DEFAULT = 1000
-      private_constant :PER_PAGE_DEFAULT
-
-      class << self
-        def declare_generic_options(options)
-          options.declare(:query, 'Additional filter for for some commands (list/delete)', types: [Hash, Array])
-          options.declare(:property, 'Name of property to set (modify operation)')
-          options.declare(:bulk, 'Bulk operation (only some)', values: :bool, default: :no)
-          options.declare(:bfail, 'Bulk operation error handling', values: :bool, default: :yes)
-        end
-      end
-
-      def initialize(context:)
-        # check presence in descendant of mandatory method and constant
-        Aspera.assert(respond_to?(:execute_action), type: InternalError){"Missing method 'execute_action' in #{self.class}"}
-        Aspera.assert(self.class.constants.include?(:ACTIONS), type: InternalError){"Missing constant 'ACTIONS' in #{self.class}"}
-        @context = context
-        add_manual_header if @context.man_header
-      end
-
-      # Global objects
-      attr_reader :context
-
-      def options; @context.options; end
-      def transfer; @context.transfer; end
-      def config; @context.config; end
-      def formatter; @context.formatter; end
-      def persistency; @context.persistency; end
-
-      def add_manual_header(has_options = true)
-        # manual header for all plugins
-        options.parser.separator('')
-        options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
-        options.parser.separator("SUBCOMMANDS: #{self.class.const_get(:ACTIONS).map(&:to_s).sort.join(' ')}")
-        options.parser.separator('OPTIONS:') if has_options
-      end
-
-      # Must be called AFTER the instance action:
-      # ... folder browse _call_instance_identifier
-      #
-      # @param description [String] description of the identifier
-      # @param as_option   [Symbol] option name to use if identifier is an option
-      # @param block       [Proc] block to search for identifier based on attribute value
-      # @return   [String, Array] identifier or list of ids
-      def instance_identifier(description: 'identifier', as_option: nil, &block)
-        if as_option.nil?
-          res_id = options.get_next_argument(description, multiple: options.get_option(:bulk)) if res_id.nil?
-        else
-          res_id = options.get_option(as_option)
-        end
-        # can be an Array
-        if res_id.is_a?(String) && (m = res_id.match(REGEX_LOOKUP_ID_BY_FIELD))
-          if block
-            res_id = yield(m[1], ExtendedValue.instance.evaluate(m[2]))
-          else
-            raise Cli::BadArgument, "Percent syntax for #{description} not supported in this context"
-          end
-        end
-        return res_id
-      end
-
-      # For create and delete operations: execute one actin or multiple if bulk is yes
-      # @param command   [Symbol] operation: :create, :delete, ...
-      # @param descr     [String] description of the value
-      # @param values    [Object] the value(s), or the type of value to get from user
-      # @param id_result [String] key in result hash to use as identifier
-      # @param fields    [Array]  fields to display
-      # @param &block    [Proc]   block to execute for each value
-      def do_bulk_operation(command:, descr: nil, values: Hash, id_result: 'id', fields: :default)
-        Aspera.assert(block_given?){'missing block'}
-        is_bulk = options.get_option(:bulk)
-        case values
-        when :identifier
-          values = instance_identifier(description: descr)
-        when Class
-          values = value_create_modify(command: command, description: descr, type: values, bulk: is_bulk)
-        end
-        # if not bulk, there is a single value
-        params = is_bulk ? values : [values]
-        Log.log.warn('Empty list given for bulk operation') if params.empty?
-        Log.dump(:bulk_operation, params)
-        result_list = []
-        params.each do |param|
-          # init for delete
-          result = {id_result => param}
-          begin
-            # execute custom code
-            res = yield(param)
-            # if block returns a hash, let's use this (create)
-            result = res if res.is_a?(Hash)
-            # TODO: remove when faspio gw api fixes this
-            result = res.first if res.is_a?(Array) && res.first.is_a?(Hash)
-            # create -> created
-            result['status'] = "#{command}#{'e' unless command.to_s.end_with?('e')}d".gsub(/yed$/, 'ied')
-          rescue StandardError => e
-            raise e if options.get_option(:bfail)
-            result['status'] = e.to_s
-          end
-          result_list.push(result)
-        end
-        display_fields = [id_result, 'status']
-        if is_bulk
-          return Main.result_object_list(result_list, fields: display_fields)
-        else
-          display_fields = fields unless fields.eql?(:default)
-          return Main.result_single_object(result_list.first, fields: display_fields)
-        end
-      end
-
-      # Operations: Create, Delete, Show, List, Modify
-      # @param api            [Rest]    api to use
-      # @param entity         [String]  sub path in URL to resource relative to base url
-      # @param command        [Symbol]  command to execute: create show list modify delete
-      # @param display_fields [Array]   fields to display by default
-      # @param items_key      [String]  result is in a sub key of the json
-      # @param delete_style   [String]  if set, the delete operation by array in payload
-      # @param id_as_arg      [String]  if set, the id is provided as url argument ?<id_as_arg>=<id>
-      # @param is_singleton   [Boolean] if true, entity is the full path to the resource
-      # @param tclo           [Bool]    if set, :list use paging with total_count, limit, offset
-      # @param block          [Proc]    block to search for identifier based on attribute value
-      # @return result suitable for CLI result
-      def entity_execute(
-        api:,
-        entity:,
-        command: nil,
-        display_fields: nil,
-        items_key: nil,
-        delete_style: nil,
-        id_as_arg: false,
-        is_singleton: false,
-        list_query: nil,
-        tclo: false,
-        &block
-      )
-        command = options.get_next_command(ALL_OPS) if command.nil?
-        if is_singleton
-          one_res_path = entity
-        elsif INSTANCE_OPS.include?(command)
-          one_res_id = instance_identifier(&block)
-          one_res_path = "#{entity}/#{one_res_id}"
-          one_res_path = "#{entity}?#{id_as_arg}=#{one_res_id}" if id_as_arg
-        end
-
-        case command
-        when :create
-          raise BadArgument, 'cannot create singleton' if is_singleton
-          return do_bulk_operation(command: command, descr: 'data', fields: display_fields) do |params|
-            api.create(entity, params)
-          end
-        when :delete
-          raise BadArgument, 'cannot delete singleton' if is_singleton
-          if !delete_style.nil?
-            one_res_id = [one_res_id] unless one_res_id.is_a?(Array)
-            Aspera.assert_type(one_res_id, Array, type: Cli::BadArgument)
-            api.call(
-              operation:    'DELETE',
-              subpath:      entity,
-              content_type: Rest::MIME_JSON,
-              body:         {delete_style => one_res_id},
-              headers:      {'Accept' => Rest::MIME_JSON}
-            )
-            return Main.result_status('deleted')
-          end
-          return do_bulk_operation(command: command, values: one_res_id) do |one_id|
-            api.delete("#{entity}/#{one_id}", query_read_delete)
-            {'id' => one_id}
-          end
-        when :show
-          return Main.result_single_object(api.read(one_res_path), fields: display_fields)
-        when :list
-          if tclo
-            data, total = list_entities_limit_offset_total_count(api: api, entity:, items_key: items_key, query: list_query)
-            return Main.result_object_list(data, total: total, fields: display_fields)
-          end
-          resp = api.call(operation: 'GET', subpath: entity, headers: {'Accept' => Rest::MIME_JSON}, query: query_read_delete)
-          return Main.result_empty if resp[:http].code == '204'
-          data = resp[:data]
-          # TODO: not generic : which application is this for ?
-          if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
-            Log.log.debug('is vnd.api')
-            data = data[entity]
-          end
-          data = data[items_key] if items_key
-          case data
-          when Hash
-            return Main.result_single_object(data, fields: display_fields)
-          when Array
-            return Main.result_object_list(data, fields: display_fields) if data.empty? || data.first.is_a?(Hash)
-            return Main.result_value_list(data)
-          else
-            raise "An error occurred: unexpected result type for list: #{data.class}"
-          end
-        when :modify
-          parameters = value_create_modify(command: command)
-          property = options.get_option(:property)
-          parameters = {property => parameters} unless property.nil?
-          api.update(one_res_path, parameters)
-          return Main.result_status('modified')
-        else
-          raise "unknown action: #{command}"
-        end
-      end
-
-      # Query parameters in URL suitable for REST: list/GET and delete/DELETE
-      def query_read_delete(default: nil)
-        query = options.get_option(:query)
-        # dup default, as it could be frozen
-        query = default.dup if query.nil?
-        Log.log.debug{"query_read_delete=#{query}".bg_red}
-        begin
-          # check it is suitable
-          URI.encode_www_form(query) unless query.nil?
-        rescue StandardError => e
-          raise Cli::BadArgument, "Query must be an extended value (Hash, Array) which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
-        end
-        return query
-      end
-
-      # Retrieves an extended value from command line, used for creation or modification of entities
-      # @param command [Symbol] command name for error message
-      # @param type [Class] expected type of value, either a Class, an Array of Class
-      # @param bulk [Boolean] if true, value must be an Array of <type>
-      # @param default [Object] default value if not provided
-      def value_create_modify(command:, description: nil, type: Hash, bulk: false, default: nil)
-        value = options.get_next_argument(
-          "parameters for #{command}#{" (#{description})" unless description.nil?}", mandatory: default.nil?,
-          validation: bulk ? Array : type
-        )
-        value = default if value.nil?
-        unless type.nil?
-          type = [type] unless type.is_a?(Array)
-          Aspera.assert(type.all?(Class)){"check types must be a Class, not #{type.map(&:class).join(',')}"}
-          if bulk
-            Aspera.assert_type(value, Array, type: Cli::BadArgument)
-            value.each do |v|
-              Aspera.assert_values(v.class, type, type: Cli::BadArgument)
-            end
-          else
-            Aspera.assert_values(value.class, type, type: Cli::BadArgument)
-          end
-        end
-        return value
-      end
-
-      # Get a (full or partial) list of all entities of a given type with query: offset/limit
-      # @param `api`       [Rest]          the API object
-      # @param `entity`    [String,Symbol] the API endpoint of entity to list
-      # @param `items_key` [String]        key in the result to get the list of items
-      # @param `query`     [Hash,nil]      additional query parameters
-      # @return [Array] items, total_count
-      def list_entities_limit_offset_total_count(
-        api:,
-        entity:,
-        items_key: nil,
-        query: nil
-      )
-        entity = entity.to_s if entity.is_a?(Symbol)
-        items_key = entity.split('/').last if items_key.nil?
-        query = {} if query.nil?
-        Aspera.assert_type(entity, String)
-        Aspera.assert_type(items_key, String)
-        Aspera.assert_type(query, Hash)
-        Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
-        result = []
-        offset = 0
-        max_items = query.delete(MAX_ITEMS)
-        remain_pages = query.delete(MAX_PAGES)
-        # merge default parameters, by default 100 per page
-        query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
-        total_count = nil
-        loop do
-          query['offset'] = offset
-          page_result = api.read(entity, query)
-          Aspera.assert_type(page_result[items_key], Array)
-          result.concat(page_result[items_key])
-          # reach the limit set by user ?
-          if !max_items.nil? && (result.length >= max_items)
-            result = result.slice(0, max_items)
-            break
-          end
-          total_count ||= page_result['total_count']
-          break if result.length >= total_count
-          remain_pages -= 1 unless remain_pages.nil?
-          break if remain_pages == 0
-          offset += page_result[items_key].length
-          formatter.long_operation_running
-        end
-        formatter.long_operation_terminated
-        return result, total_count
-      end
-
-      # Lookup an entity id from its name
-      # @param entity    [String] the type of entity to lookup, by default it is the path, and it is also the field name in result
-      # @param value     [String] the value to lookup
-      # @param field     [String] the field to match, by default it is 'name'
-      # @param items_key [String] key in the result to get the list of items (override entity)
-      # @param query     [Hash]   additional query parameters
-      def lookup_entity_by_field(api:, entity:, value:, field: 'name', items_key: nil, query: :default)
-        if query.eql?(:default)
-          Aspera.assert(field.eql?('name')){'Default query is on name only'}
-          query = {'q'=> value}
-        end
-        found = list_entities_limit_offset_total_count(api: api, entity: entity, items_key: items_key, query: query).first.select{ |i| i[field].eql?(value)}
-        case found.length
-        when 0 then raise "No #{entity} with #{field} = #{value}"
-        when 1 then return found.first
-        else raise "Found #{found.length} #{entity} with #{field} = #{value}"
-        end
-      end
-    end
-  end
-end