aspera-cli 4.25.4 → 4.25.5

data/lib/aspera/api/node.rb

@@ -14,7 +14,8 @@ require 'net/ssh/buffer'
 
 module Aspera
   module Api
-    # Provides additional functions using node API with gen4 extensions (access keys)
+    # Aspera Node API client
+    # with gen4 extensions (access keys)
     class Node < Rest
       # Format of node scope : node.<access key>:<scope>
       module Scope
@@ -508,7 +509,7 @@ module Aspera
           # Get the transfer user from info on access key
           transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
           # Get settings from name.value array to hash key.value
-          settings = info['settings']&.each_with_object({}){ |i, h| h[i['name']] = i['value']}
+          settings = info['settings']&.to_h{ |i| [i['name'], i['value']]}
           # Check WSS ports
           Transfer::Spec::WSS_FIELDS.each do |i|
             transfer_spec[i] = settings[i] if settings.key?(i)
@@ -520,6 +521,46 @@ module Aspera
         return transfer_spec
       end
 
+      # Executes `GET` call in loop using `iteration_token` (`/ops/transfers`)
+      # @param iteration [Array]   a single element array with the iteration token or nil
+      # @param call_args [Hash]    additional arguments to pass to `Rest.call`
+      # @return [Array] list of items returned by the API call
+      def read_with_paging(subpath, query = nil, iteration: nil, **call_args)
+        Aspera.assert_type(iteration, Array, NilClass){'iteration'}
+        Aspera.assert_type(query, Hash, NilClass){'query'}
+        Aspera.assert(!call_args.key?(:query))
+        query = {} if query.nil?
+        query[:iteration_token] = iteration[0] unless iteration.nil?
+        max = query.delete(RestList::MAX_ITEMS)
+        item_list = []
+        loop do
+          data, http = read(subpath, query, **call_args, ret: :both)
+          Aspera.assert_type(data, Array){"Expected data to be an Array, got: #{data.class}"}
+          # no data
+          break if data.empty?
+          # get next iteration token from link
+          next_iteration_token = nil
+          link_info = http['Link']
+          unless link_info.nil?
+            m = link_info.match(/<([^>]+)>/)
+            Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
+            next_iteration_token = Rest.query_to_h(URI.parse(m[1]).query)['iteration_token']
+          end
+          # same as last iteration: stop
+          break if next_iteration_token&.eql?(query[:iteration_token])
+          query[:iteration_token] = next_iteration_token
+          item_list.concat(data)
+          if max&.<=(item_list.length)
+            item_list = item_list.slice(0, max)
+            break
+          end
+          break if next_iteration_token.nil?
+        end
+        # save iteration token if needed
+        iteration[0] = query[:iteration_token] unless iteration.nil?
+        item_list
+      end
+
       private
 
       # Method called in loop for each entry for `resolve_api_fid`

data/lib/aspera/ascp/installation.rb

@@ -83,16 +83,15 @@ module Aspera
 
       # @return [Hash] with key = file name (String), and value = path to file
       def file_paths
-        return SDK_FILES.each_with_object({}) do |v, m|
-          m[v.to_s] =
-            begin
-              path(v)
-            rescue Errno::ENOENT => e
-              e.message.gsub(/.*assertion failed: /, '').gsub(/\): .*/, ')')
-            rescue => e
-              e.message
-            end
-        end
+        return SDK_FILES.to_h do |v|
+                 [v.to_s, begin
+                   path(v)
+                 rescue Errno::ENOENT => e
+                   e.message.gsub(/.*assertion failed: /, '').gsub(/\): .*/, ')')
+                 rescue => e
+                   e.message
+                 end]
+               end
       end
 
       # TODO: if using another product than SDK, should use files from there

data/lib/aspera/cli/error.rb

@@ -9,7 +9,15 @@ module Aspera
     class MissingArgument < Error; end
     class NoSuchElement < Error; end
 
+    # Raised when a lookup for a specific entity fails to return exactly one result.
+    #
+    # Provides a formatted message indicating whether the entity was missing
+    # or if multiple matches were found (ambiguity).
     class BadIdentifier < Error
+      # @param res_type [String] The type of entity being looked up (e.g., 'user').
+      # @param res_id   [String] The value of the identifier that failed.
+      # @param field    [String] The name of the field used for lookup (defaults to 'identifier').
+      # @param count    [Integer] The number of matches found (0 for not found, >1 for ambiguous).
       def initialize(res_type, res_id, field: 'identifier', count: 0)
         msg = count.eql?(0) ? 'not found' : "found #{count}"
         super("#{res_type} with #{field}=#{res_id}: #{msg}")

data/lib/aspera/cli/info.rb

@@ -7,7 +7,8 @@ module Aspera
       CMD_NAME = 'ascli'
       # Name of the containing gem, same as in <gem name>.gemspec
       GEM_NAME = 'aspera-cli'
-      DOC_URL  = "https://www.rubydoc.info/gems/#{GEM_NAME}"
+      DOC_URL  = 'https://ibm.biz/ascli-doc'
+      RUBYDOC_URL = "https://www.rubydoc.info/gems/#{GEM_NAME}"
       GEM_URL  = "https://rubygems.org/gems/#{GEM_NAME}"
       SRC_URL  = 'https://github.com/IBM/aspera-cli'
       CONTAINER = 'docker.io/martinlaurent/ascli'

data/lib/aspera/cli/main.rb

@@ -12,18 +12,34 @@ require 'aspera/cli/hints'
 require 'aspera/secret_hider'
 require 'aspera/log'
 require 'aspera/assert'
+require 'net/ssh/errors'
+require 'openssl'
 
 module Aspera
   module Cli
     # Global objects shared with plugins
     class Context
+      # @!attribute [rw] options
+      #   @return [Manager] the command line options manager
+      # @!attribute [rw] transfer
+      #   @return [TransferAgent] the transfer agent, used by transfer plugins
+      # @!attribute [rw] config
+      #   @return [Plugins::Config] the configuration plugin, used by plugins to get configuration values and presets
+      # @!attribute [rw] formatter
+      #   @return [Formatter] the formatter, used by plugins to display results and messages
+      # @!attribute [rw] persistency
+      #   @return [Object] # whatever the type is
+      # @!attribute [rw] man_header
+      #   @return [Boolean] whether to display the manual header in plugin help
       MEMBERS = %i[options transfer config formatter persistency man_header].freeze
       attr_accessor(*MEMBERS)
 
+      # Initialize all members to nil, so that they are defined and can be validated later
       def initialize
-        @man_header = true
+        MEMBERS.each{ |i| instance_variable_set(:"@#{i}", nil)}
       end
 
+      # Validate that all members are set, raise exception if not
       def validate
         MEMBERS.each do |i|
           Aspera.assert(instance_variable_defined?(:"@#{i}"))
@@ -35,7 +51,7 @@ module Aspera
         transfer.eql?(:only_manual)
       end
 
-      def only_manual
+      def only_manual!
         @transfer = :only_manual
       end
     end
@@ -89,13 +105,15 @@ module Aspera
           status_table.each do |item|
             worst = TransferAgent.session_status(item[STATUS_FIELD])
             global_status = worst unless worst.eql?(:success)
-            item[STATUS_FIELD] = item[STATUS_FIELD].map(&:to_s).join(',')
+            item[STATUS_FIELD] = item[STATUS_FIELD].join(',')
           end
           raise global_status unless global_status.eql?(:success)
           return result_object_list(status_table)
         end
 
         # Display image for that URL or directly blob
+        #
+        # @param url_or_blob [String] URL or blob to display as image
         def result_image(url_or_blob)
           return {type: :image, data: url_or_blob}
         end
@@ -111,6 +129,9 @@ module Aspera
         end
 
         # A list of values
+        #
+        # @param data [Array] The list of values
+        # @param name [String] The name of the list (used for display)
         def result_value_list(data, name: 'id')
           Aspera.assert_type(data, Array)
           Aspera.assert_type(name, String)
@@ -155,7 +176,9 @@ module Aspera
         execute_command = true
         # Catch exceptions
         begin
-          init_agents_options_plugins
+          init_agents_and_options
+          # Find plugins, shall be after parse! ?
+          Plugins::Factory.instance.add_plugins_from_lookup_folders
           # Help requested without command ? (plugins must be known here)
           show_usage if @option_help && @context.options.command_or_arg_empty?
           generate_bash_completion if @bash_completion
@@ -253,17 +276,11 @@ module Aspera
         return
       end
 
-      def init_agents_options_plugins
-        init_agents_and_options
-        # Find plugins, shall be after parse! ?
-        Plugins::Factory.instance.add_plugins_from_lookup_folders
-      end
-
       def show_usage(all: true, exit: true)
         # Display main plugin options (+config)
         @context.formatter.display_message(:error, @context.options.parser)
         if all
-          @context.only_manual
+          @context.only_manual!
           # List plugins that have a "require" field, i.e. all but main plugin
           Plugins::Factory.instance.plugin_list.each do |plugin_name_sym|
             # Config was already included in the global options
@@ -283,6 +300,7 @@ module Aspera
 
       # This can throw exception if there is a problem with the environment, needs to be caught by execute method
       def init_agents_and_options
+        @context.man_header = true
         # Create formatter, in case there is an exception, it is used to display.
         @context.formatter = Formatter.new
         # Create command line manager with arguments
@@ -336,7 +354,7 @@ module Aspera
           OPTIONS
           #{t}Options begin with a '-' (minus), and value is provided on command line.
           #{t}Special values are supported beginning with special prefix @pfx:, where pfx is one of:
-          #{t}#{ExtendedValue.instance.modifiers.map(&:to_s).join(', ')}
+          #{t}#{ExtendedValue.instance.modifiers.join(', ')}
           #{t}Dates format is 'DD-MM-YY HH:MM:SS', or 'now' or '-<num>h'
 
           ARGS

data/lib/aspera/cli/manager.rb

@@ -194,7 +194,9 @@ module Aspera
           [error_msg, 'Use:'].concat(accept_list.map{ |c| "- #{c}"}.sort).join("\n")
         end
 
-        # change option name with dash to name with underscore
+        # Change option name with dash to name with underscore
+        # @param name [String] option name
+        # @return [String]
         def option_line_to_name(name)
           return name.gsub(OPTION_SEP_LINE, OPTION_SEP_SYMBOL)
         end
@@ -211,7 +213,7 @@ module Aspera
       def initialize(program_name, argv = nil)
         # command line values *not* starting with '-'
         @unprocessed_cmd_line_arguments = []
-        # command line values starting with '-'
+        # command line values starting with at least one '-'
         @unprocessed_cmd_line_options = []
         # a copy of all initial options
         @initial_cli_options = []
@@ -330,7 +332,7 @@ module Aspera
 
       # @param descr       [String] description for help
       # @param mandatory   [Boolean] if true, raise error if option not set
-      # @param multiple    [Boolean] if true, return remaining arguments (Array) unil END
+      # @param multiple    [Boolean] if true, return remaining arguments (Array) until END
       # @param accept_list [Array, NilClass] list of allowed values (Symbol)
       # @param validation  [Class, Array, NilClass] Accepted value type(s) or list of Symbols
       # @param aliases     [Hash] map of aliases: key = alias, value = real value
@@ -466,21 +468,17 @@ module Aspera
       # @return [Hash] options as taken from config file and command line just before command execution
       def unprocessed_options_with_value
         result = {}
-        @initial_cli_options.each do |option_value|
-          case option_value
-          when /^#{OPTION_PREFIX}([^=]+)$/o
-            # ignore
-          when /^#{OPTION_PREFIX}([^=]+)=(.*)$/o
-            name = Regexp.last_match(1)
-            value = Regexp.last_match(2)
-            name.gsub!(OPTION_SEP_LINE, OPTION_SEP_SYMBOL)
-            value = ExtendedValue.instance.evaluate(value, context: "option: #{name}")
-            Log.log.debug{"option #{name}=#{value}"}
-            result[name] = value
-            @unprocessed_cmd_line_options.delete(option_value)
-          else
-            raise Cli::BadArgument, "wrong option format: #{option_value}"
-          end
+        @initial_cli_options.each do |option_argument|
+          # ignore short options
+          next unless option_argument.start_with?(OPTION_PREFIX)
+          name, value = option_argument[OPTION_PREFIX.length..-1].split(OPTION_VALUE_SEPARATOR, 2)
+          # ignore options without value
+          next if value.nil?
+          Log.log.debug{"option #{name}=#{value}"}
+          path = name.split(DotContainer::SEPARATOR)
+          path[0] = self.class.option_line_to_name(path[0])
+          DotContainer.dotted_to_container(path, smart_convert(value), result)
+          @unprocessed_cmd_line_options.delete(option_argument)
         end
         return result
       end
@@ -516,20 +514,25 @@ module Aspera
         rescue OptionParser::InvalidOption => e
           Log.log.trace1{"InvalidOption #{e}".red}
           # An option like --a.b.c=d does: a={"b":{"c":ext_val(d)}}
-          if (m = e.args.first.match(/^--([a-z\-]+)\.([^=]+)=(.+)$/))
-            option, path, value = m.captures
-            option_sym = self.class.option_line_to_name(option).to_sym
-            if @declared_options.key?(option_sym)
-              set_option(option_sym, DotContainer.dotted_to_container(path, smart_convert(value), get_option(option_sym)), where: 'dotted')
-              retry
+          if e.args.first.start_with?(OPTION_PREFIX)
+            name, value = e.args.first[OPTION_PREFIX.length..-1].split(OPTION_VALUE_SEPARATOR, 2)
+            if !value.nil?
+              path = name.split(DotContainer::SEPARATOR)
+              option_sym = self.class.option_line_to_name(path.shift).to_sym
+              if @declared_options.key?(option_sym)
+                # it's a known option, so let's process it
+                set_option(option_sym, DotContainer.dotted_to_container(path, smart_convert(value), get_option(option_sym)), where: 'dotted')
+                # resume to next
+                retry
+              end
             end
           end
-          # save for later processing
+          # Save for later processing
           unknown_options.push(e.args.first)
           retry
         end
         Log.log.trace1{"remains: #{unknown_options}"}
-        # set unprocessed options for next time
+        # Set unprocessed options for next time
         @unprocessed_cmd_line_options = unknown_options
       end
 
@@ -596,9 +599,9 @@ module Aspera
         ExtendedValue.assert_no_value(arg, :p)
         result = nil
         get_next_argument(:args, multiple: true).each do |arg|
-          Aspera.assert(arg.include?(OPTION_VALUE_SEPARATOR)){"Positional argument: #{arg} does not inlude #{OPTION_VALUE_SEPARATOR}"}
+          Aspera.assert(arg.include?(OPTION_VALUE_SEPARATOR)){"Positional argument: #{arg} does not include #{OPTION_VALUE_SEPARATOR}"}
           path, value = arg.split(OPTION_VALUE_SEPARATOR, 2)
-          result = DotContainer.dotted_to_container(path, smart_convert(value), result)
+          result = DotContainer.dotted_to_container(path.split(DotContainer::SEPARATOR), smart_convert(value), result)
         end
         result
       end

data/lib/aspera/cli/plugins/aoc.rb

@@ -744,7 +744,7 @@ module Aspera
         # Short link entity: `short_links` have:
         # - a numerical id, e.g. `764412`
         # - a resource type, e.g. `UrlToken`
-        # - a ressource id, e.g. `scQ7uXPbvQ`
+        # - a resource id, e.g. `scQ7uXPbvQ`
         # - a short URL path, e.g. `dxyRpT9`
         # @param shared_data [Hash] Information for shared data: dropbox_id+name or file_id+node_id
         # @param &perm_block [Proc] Optional: create/modify/delete permissions on node
@@ -887,7 +887,7 @@ module Aspera
 
         def reject_packages_from_persistency(all_packages, skip_ids_persistency)
           return if skip_ids_persistency.nil?
-          skip_package = skip_ids_persistency.data.each_with_object({}){ |i, m| m[i] = true}
+          skip_package = skip_ids_persistency.data.to_h{ |i| [i, true]}
           all_packages.reject!{ |pkg| skip_package[pkg['id']]}
         end
 

data/lib/aspera/cli/plugins/base.rb

@@ -14,10 +14,6 @@ module Aspera
         INSTANCE_OPS = %i[modify delete show].freeze
         # All standard operations (create list modify delete show)
         ALL_OPS = (GLOBAL_OPS + INSTANCE_OPS).freeze
-        # Special query parameter: `max`: max number of items for list command
-        MAX_ITEMS = 'max'
-        # Special query parameter: `pmax`: max number of pages for list command
-        MAX_PAGES = 'pmax'
 
         class << self
           def declare_options(options)
@@ -138,7 +134,6 @@ module Aspera
         # @param delete_style   [String]  If set, the delete operation by array in payload
         # @param id_as_arg      [String]  If set, the id is provided as url argument ?<id_as_arg>=<id>
         # @param is_singleton   [Boolean] If `true`, entity is the full path to the resource
-        # @param tclo           [Boolean] If `true`, :list use paging with total_count, limit, offset
         # @param block          [Proc]    Block to search for identifier based on attribute value
         # @return [Hash] Result suitable for CLI result
         def entity_execute(
@@ -151,7 +146,6 @@ module Aspera
           id_as_arg: false,
           is_singleton: false,
           list_query: nil,
-          tclo: false,
           &block
         )
           command = options.get_next_command(ALL_OPS) if command.nil?
@@ -189,10 +183,6 @@ module Aspera
           when :show
             return Main.result_single_object(api.read(one_res_path), fields: display_fields)
           when :list
-            if tclo
-              data, total = list_entities_limit_offset_total_count(api: api, entity:, items_key: items_key, query: query_read_delete(default: list_query))
-              return Main.result_object_list(data, total: total, fields: display_fields)
-            end
             data, http = api.read(entity, query_read_delete, ret: :both)
             return Main.result_empty if http.code == '204'
             # TODO: not generic : which application is this for ?
@@ -259,86 +249,9 @@ module Aspera
           return value
         end
 
-        # Get a (full or partial) list of all entities of a given type with query: offset/limit
-        # @param api       [Rest]          API object
-        # @param entity    [String,Symbol] API endpoint of entity to list
-        # @param items_key [String]        Key in the result to get the list of items (Default: same as `entity`)
-        # @param query     [Hash,nil]      Additional query parameters
-        # @return [Array<(Array<Hash>, Integer)>] items, total_count
-        def list_entities_limit_offset_total_count(
-          api:,
-          entity:,
-          items_key: nil,
-          query: nil
-        )
-          entity = entity.to_s if entity.is_a?(Symbol)
-          items_key = entity.split('/').last if items_key.nil?
-          query = {} if query.nil?
-          Aspera.assert_type(entity, String)
-          Aspera.assert_type(items_key, String)
-          Aspera.assert_type(query, Hash)
-          Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
-          result = []
-          offset = 0
-          max_items = query.delete(MAX_ITEMS)
-          remain_pages = query.delete(MAX_PAGES)
-          # Merge default parameters, by default 100 per page
-          query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
-          total_count = nil
-          loop do
-            query['offset'] = offset
-            page_result = api.read(entity, query)
-            Aspera.assert_type(page_result[items_key], Array)
-            result.concat(page_result[items_key])
-            # Reach the limit set by user ?
-            if !max_items.nil? && (result.length >= max_items)
-              result = result.slice(0, max_items)
-              break
-            end
-            total_count ||= page_result['total_count']
-            break if result.length >= total_count
-            remain_pages -= 1 unless remain_pages.nil?
-            break if remain_pages == 0
-            offset += page_result[items_key].length
-            RestParameters.instance.spinner_cb.call("#{result.length} / #{total_count || '?'}")
-          end
-          RestParameters.instance.spinner_cb.call(action: :success)
-          return result, total_count
-        end
-
-        # Lookup an entity id from its name.
-        # Uses query `q` if `query` is `:default` and `field` is `name`.
-        # @param entity    [String] Type of entity to lookup, by default it is the path, and it is also the field name in result
-        # @param value     [String] Value to lookup
-        # @param field     [String] Field to match, by default it is `'name'`
-        # @param items_key [String] Key in the result to get the list of items (override entity)
-        # @param query     [Hash]   Additional query parameters (Default: `:default`)
-        def lookup_entity_by_field(api:, entity:, value:, field: 'name', items_key: nil, query: :default)
-          if query.eql?(:default)
-            Aspera.assert(field.eql?('name')){'Default query is on name only'}
-            query = {'q'=> value}
-          end
-          lookup_entity_generic(entity: entity, field: field, value: value){list_entities_limit_offset_total_count(api: api, entity: entity, items_key: items_key, query: query).first}
-        end
-
-        # Lookup entity by field and value. Extract single result from list of result returned by block.
-        # @param entity    [String] Type of entity to lookup, by default it is the path, and it is also the field name in result
-        # @param value     [String] Value to lookup
-        # @param field     [String] Field to match, by default it is `'name'`
-        # @param block     [Proc]   Get list of entity matching query.
-        def lookup_entity_generic(entity:, value:, field: 'name', &block)
-          Aspera.assert(block_given?)
-          found = yield
-          Aspera.assert_array_all(found, Hash)
-          found = found.select{ |i| i[field].eql?(value)}
-          return found.first if found.length.eql?(1)
-          raise Cli::BadIdentifier.new(entity, value, field: field, count: found.length)
-        end
-
-        PER_PAGE_DEFAULT = 1000
         # Percent selector: select by this field for this value
         REGEX_LOOKUP_ID_BY_FIELD = /^%([^:]+):(.*)$/
-        private_constant :PER_PAGE_DEFAULT, :REGEX_LOOKUP_ID_BY_FIELD
+        private_constant :REGEX_LOOKUP_ID_BY_FIELD
       end
     end
   end

data/lib/aspera/cli/plugins/faspex.rb

@@ -9,6 +9,7 @@ require 'aspera/cli/transfer_agent'
 require 'aspera/transfer/uri'
 require 'aspera/transfer/spec'
 require 'aspera/persistency_action_once'
+require 'aspera/rest_list'
 require 'aspera/environment'
 require 'aspera/nagios'
 require 'aspera/id_generator'
@@ -31,7 +32,7 @@ module Aspera
         # allowed parameters for inbox.atom
         ATOM_PARAMS = %w[page count startIndex].freeze
         # with special parameters (from Plugin class) : max and pmax (from Plugin)
-        ATOM_EXT_PARAMS = [MAX_ITEMS, MAX_PAGES].concat(ATOM_PARAMS).freeze
+        ATOM_EXT_PARAMS = [RestList::MAX_ITEMS, RestList::MAX_PAGES].concat(ATOM_PARAMS).freeze
         # sub path in url for public link delivery
         PUB_LINK_EXTERNAL_MATCH = 'external_deliveries/'
         STANDARD_PATH = '/aspera/faspex'
@@ -176,10 +177,9 @@ module Aspera
             Aspera.assert_type(mailbox_query, Hash){'query'}
             Aspera.assert((mailbox_query.keys - ATOM_EXT_PARAMS).empty?){"query: supported params: #{ATOM_EXT_PARAMS}"}
             Aspera.assert(!(mailbox_query.key?('startIndex') && mailbox_query.key?('page'))){'query: startIndex and page are exclusive'}
-            max_items = mailbox_query[MAX_ITEMS]
-            mailbox_query.delete(MAX_ITEMS)
-            max_pages = mailbox_query[MAX_PAGES]
-            mailbox_query.delete(MAX_PAGES)
+            # Extract pagination control parameters (not part of API query)
+            max_items = mailbox_query.delete(RestList::MAX_ITEMS)
+            max_pages = mailbox_query.delete(RestList::MAX_PAGES)
           end
           loop do
             # get a batch of package information
@@ -211,7 +211,7 @@ module Aspera
                 end
               # add special key
               package['items'] = package['link'].is_a?(Array) ? package['link'].length : 0
-              package['metadata'] = package['metadata']['field'].each_with_object({}){ |i, m| m[i['name']] = i['content']}
+              package['metadata'] = package['metadata']['field'].to_h{ |i| [i['name'], i['content']]}
               # if we look for a specific package
               stop_condition = true if !stop_at_id.nil? && stop_at_id.eql?(package[PACKAGE_MATCH_FIELD])
               # keep only those for the specified recipient