aspera-cli 4.25.2 → 4.25.4

data/lib/aspera/api/node.rb

@@ -6,16 +6,16 @@ require 'aspera/oauth'
 require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/environment'
-require 'zlib'
 require 'base64'
 require 'openssl'
 require 'pathname'
+require 'zlib'
 require 'net/ssh/buffer'
 
 module Aspera
   module Api
     # Provides additional functions using node API with gen4 extensions (access keys)
-    class Node < Aspera::Rest
+    class Node < Rest
       # Format of node scope : node.<access key>:<scope>
       module Scope
         # Node sub-scopes
@@ -42,9 +42,10 @@ module Aspera
       ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
       # Special HTTP Headers
       HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
-      HEADER_X_TOTAL_COUNT = 'X-Total-Count'
       HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
+      HEADER_X_TOTAL_COUNT = 'X-Total-Count'
       HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
+      HEADER_ACCEPT_VERSION = 'Accept-Version'
       # / in cloud
       PATH_SEPARATOR = '/'
 
@@ -52,22 +53,36 @@ module Aspera
       OAuth::Factory.instance.register_decoder(lambda{ |token| Node.decode_bearer_token(token)})
 
       # Class instance variable, access with accessors on class
-      @use_standard_ports = true
-      @use_node_cache = true
-
-      class << self
+      @api_options = {
         # Set to false to read transfer parameters from download_setup
-        attr_accessor :use_standard_ports
+        standard_ports: true,
         # Set to false to bypass cache in redis
-        attr_accessor :use_node_cache
+        cache:          true,
+        accept_v4:      true
+      }
+      OPTIONS = @api_options.keys.freeze
+
+      class << self
+        attr_reader :api_options
         attr_reader :use_dynamic_key
 
-        # Adds cache control header, as globally specified to read request
-        # Use like this: read(...,**cache_control)
-        def cache_control
-          headers = {'Accept' => Rest::MIME_JSON}
-          headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
-          {headers: headers}
+        def api_options=(h)
+          Aspera.assert_type(h, Hash)
+          h.each do |k, v|
+            Aspera.assert(@api_options.key?(k.to_sym)){"unknown api option: #{k} (#{OPTIONS.join(', ')})"}
+            Aspera.assert_type(v, TrueClass, FalseClass){"api options value for #{k} should be boolean"}
+            @api_options[k.to_sym] = v
+          end
+        end
+
+        # Adds cache control header for node API /files/:id
+        # as globally specified to read request
+        # Use like this: read(..., headers: add_cache_control)
+        # @param headers [Hash] optional initial headers to add to
+        # @return [Hash] headers with cache control header added if needed
+        def add_cache_control(headers = {})
+          headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless api_options[:cache]
+          headers
         end
 
         # Set private key to be used
@@ -180,8 +195,8 @@ module Aspera
             Aspera.assert(!access_key.nil?)
           end
           return {
-            Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
-            'Authorization'                  => bearer_auth
+            HEADER_X_ASPERA_ACCESS_KEY => access_key,
+            'Authorization'            => bearer_auth
           }
         end
       end
@@ -248,6 +263,50 @@ module Aspera
         return false
       end
 
+      # Read folder content, with pagination management for gen4, not recursive
+      # if `Accept-Version: 4.0` is not specified:
+      #     if `page` and `per_page` are not specified, then all entries are returned.
+      #     if either `page` or `per_page` is specified, then both are required, else 400
+      # if `Accept-Version: 4.0` is specified:
+      #     those queries are not available: page (not mentioned), sort, min_size, max_size, min_modified_time, max_modified_time, target_id, target_node_id, files_prefetch_count, page, name_iglob : either ignored or result in API error 400.
+      #     query include is accepted, but seems to do nothing as access_levels and recursive_counts are already included in results.
+      #     query `iteration_token` is accepted and allows to get paginated results, with `X-Aspera-Next-Iteration-Token` header in response to get next page token. `X-Aspera-Total-Count` header gives total count of entries.
+      def read_folder_content(file_id, query = nil, exception: true, path: nil)
+        folder_items = []
+        begin
+          query ||= {}
+          headers = self.class.add_cache_control
+          use_v4 = self.class.api_options[:accept_v4]
+          return read("files/#{file_id}/files", query, headers: headers) unless use_v4 || query.key?('page') || query.key?('per_page')
+          if use_v4
+            headers[HEADER_ACCEPT_VERSION] = '4.0'
+            query['per_page'] = 1000 unless query.key?('per_page')
+          elsif query.key?('per_page') && !query.key?('page')
+            query['page'] = 0
+          end
+          loop do
+            RestParameters.instance.spinner_cb.call(folder_items.count)
+            data, http = read("files/#{file_id}/files", query, headers: headers, ret: :both)
+            folder_items.concat(data)
+            if use_v4
+              iteration_token = http[HEADER_X_NEXT_ITER_TOKEN]
+              break if iteration_token.nil? || iteration_token.empty?
+              query['iteration_token'] = iteration_token
+            else
+              break if data['item_count'].eql?(0)
+              query['offset'] += data['item_count']
+            end
+          end
+        rescue StandardError => e
+          raise e if exception
+          Log.log.warn{"#{path}: #{e.class} #{e.message}"}
+          Log.log.debug{(['Backtrace:'] + e.backtrace).join("\n")}
+        ensure
+          RestParameters.instance.spinner_cb.call(folder_items.count, action: :success)
+        end
+        folder_items
+      end
+
       # Recursively browse in a folder (with non-recursive method)
       # Entries of folders are processed if the processing method returns true
       # Links are processed on the respective node
@@ -267,14 +326,7 @@ module Aspera
           current_item = folders_to_explore.shift
           Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
           # Get folder content
-          folder_contents =
-            begin
-              # TODO: use header
-              read("files/#{current_item[:id]}/files", query, **self.class.cache_control)
-            rescue StandardError => e
-              Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
-              []
-            end
+          folder_contents = read_folder_content(current_item[:id], query, exception: false, path: current_item[:path])
           Log.dump(:folder_contents, folder_contents)
           folder_contents.each do |entry|
             if entry.key?('error')
@@ -308,7 +360,9 @@ module Aspera
       # @param top_file_id [String] id initial file id
       # @param path [String] file or folder path (end with "/" is like setting process_last_link)
       # @param process_last_link [Boolean] if true, follow the last link
-      # @return [Hash] {.api,.file_id}
+      # @return [Hash] Result data
+      # @option return [Rest] :api     REST client instance
+      # @option return [String]       :file_id File identifier
       def resolve_api_fid(top_file_id, path, process_last_link = false)
         Aspera.assert_type(top_file_id, String)
         Aspera.assert_type(path, String)
@@ -443,7 +497,7 @@ module Aspera
         # Add application specific tags (AoC)
         @app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: @app_info) unless @app_info.nil?
         # Add remote host info
-        if self.class.use_standard_ports
+        if self.class.api_options[:standard_ports]
           # Get default TCP/UDP ports and transfer user
           transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
           # By default: same address as node API

data/lib/aspera/assert.rb

@@ -93,7 +93,7 @@ module Aspera
     # The value is not one of the expected values
     # @param value  [Object]           The wrong value
     # @param type   [Exception,Symbol] Exception to raise, or Symbol for Log.log
-    # @param block  [Proc]             Additional description in front of message
+    # @param &block [Proc]             Additional description in front of message
     def error_unexpected_value(value, type: InternalError)
       report_error(type, "#{"#{yield}: " if block_given?}unexpected value: #{value.inspect}")
     end

data/lib/aspera/cli/formatter.rb

@@ -128,20 +128,34 @@ module Aspera
         @spinner = nil
       end
 
-      # call this after REST calls if several api calls are expected
-      def long_operation_running(title = '')
+      def long_operation(title = nil, action: :spin)
         return unless Environment.terminal?
-        if @spinner.nil?
+        return if %i[error data].include?(@options[:display])
+
+        # Handle the "delayed start" state
+        return @spinner = :starting if action == :spin && @spinner.nil?
+
+        # Cleanup if we try to stop a spinner that never actually started
+        @spinner = nil if action != :spin && @spinner == :starting
+        return if @spinner.nil?
+
+        # Initialize the real TTY object if it's currently just the :starting symbol
+        if @spinner == :starting
           @spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
+          @spinner.update(title: '')
           @spinner.start
         end
-        @spinner.update(title: title)
-        @spinner.spin
-      end
 
-      def long_operation_terminated
-        @spinner&.stop
-        @spinner = nil
+        @spinner.update(title: title) if title
+
+        case action
+        when :spin
+          @spinner.spin
+        when :success, :fail
+          action == :success ? @spinner.success : @spinner.error
+          @spinner.stop
+          @spinner = nil
+        end
       end
 
       def declare_options(options)
@@ -150,9 +164,9 @@ module Aspera
         else
           {}
         end
+        options.declare(:display, 'Output only some information', allowed: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :data)
         options.declare(:format, 'Output format', allowed: DISPLAY_FORMATS, handler: {o: self, m: :option_handler}, default: :table)
         options.declare(:output, 'Destination for results', handler: {o: self, m: :option_handler})
-        options.declare(:display, 'Output only some information', allowed: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :info)
         options.declare(
           :fields, "Comma separated list of: fields, or #{SpecialValues::ALL}, or #{SpecialValues::DEF}", handler: {o: self, m: :option_handler},
           allowed: [String, Array, Regexp, Proc],
@@ -178,6 +192,8 @@ module Aspera
           @options[option_symbol] = value
           # special handling of some options
           case option_symbol
+          when :format
+            @options[:display] = value.eql?(:table) ? :info : :data
           when :output
             $stdout = if value.eql?('-')
               STDOUT # rubocop:disable Style/GlobalStdStream
@@ -197,7 +213,7 @@ module Aspera
         nil
       end
 
-      # main output method
+      # Main output method
       # data: for requested data, not displayed if level==error
       # info: additional info, displayed if level==info
       # error: always displayed on stderr

data/lib/aspera/cli/hints.rb

@@ -115,6 +115,13 @@ module Aspera
           remediation: [
             'Check your public key in your AoC user profile.'
           ]
+        },
+        {
+          exception:   Aspera::RestCallError,
+          match:       /Please configure ACLs for this URI/,
+          remediation: [
+            'server must have: asnodeadmin -mu <node user> --acl-add=internal --internal'
+          ]
         }
       ]
       private_constant :ERROR_HINTS

data/lib/aspera/cli/manager.rb

@@ -12,6 +12,37 @@ require 'optparse'
 
 module Aspera
   module Cli
+    module BoolValue
+      # boolean options are set to true/false from the following values
+      YES_SYM = :yes
+      NO_SYM = :no
+      FALSE_VALUES = [NO_SYM, false].freeze
+      TRUE_VALUES = [YES_SYM, true].freeze
+      private_constant :YES_SYM, :NO_SYM, :FALSE_VALUES, :TRUE_VALUES
+      # Boolean values
+      # @return [Array<true, false, :yes, :no>]
+      ALL = (TRUE_VALUES + FALSE_VALUES).freeze
+      TYPES = [FalseClass, TrueClass].freeze
+      SYMBOLS = [NO_SYM, YES_SYM].freeze
+      # @return `true` if value is a value for `true` in ALL
+      def true?(enum)
+        Aspera.assert_values(enum, ALL){'boolean'}
+        return TRUE_VALUES.include?(enum)
+      end
+
+      # @return [:yes, :no]
+      def to_sym(enum)
+        Aspera.assert_values(enum, ALL){'boolean'}
+        return TRUE_VALUES.include?(enum) ? YES_SYM : NO_SYM
+      end
+
+      # @return `true` if value is a value for `true` or `false` in ALL
+      def symbol?(sym)
+        return ALL.include?(sym)
+      end
+      module_function :true?, :to_sym, :symbol?
+    end
+
     # Constants to be used as parameter `allowed:` for `OptionValue`
     module Allowed
       # This option can be set to a single string or array, multiple times, and gives Array of String
@@ -20,7 +51,7 @@ module Aspera
       TYPES_SYMBOL_ARRAY = [Array, Symbol].freeze
       # Value will be coerced to int
       TYPES_INTEGER = [Integer].freeze
-      TYPES_BOOLEAN = [FalseClass, TrueClass].freeze
+      TYPES_BOOLEAN = BoolValue::TYPES
       # no value at all, it's a switch
       TYPES_NONE = [].freeze
       TYPES_ENUM = [Symbol].freeze
@@ -74,7 +105,7 @@ module Aspera
             @values = allowed[Allowed::TYPES_SYMBOL_ARRAY.length..-1]
           elsif allowed.all?(Class)
             @types = allowed
-            @values = Manager::BOOLEAN_VALUES if allowed.eql?(Allowed::TYPES_BOOLEAN)
+            @values = BoolValue::ALL if allowed.eql?(Allowed::TYPES_BOOLEAN)
             # Default value for array
             @object ||= [] if @types.first.eql?(Array) && !@types.include?(NilClass)
             @object ||= {} if @types.first.eql?(Hash) && !@types.include?(NilClass)
@@ -110,7 +141,7 @@ module Aspera
         Aspera.assert(!@deprecation, type: warn){"Option #{@option} is deprecated: #{@deprecation}"}
         new_value = ExtendedValue.instance.evaluate(value, context: "option: #{@option}", allowed: @types)
         Log.log.trace1{"#{where}: #{@option} <- (#{new_value.class})#{new_value}"}
-        new_value = Manager.enum_to_bool(new_value) if @types.eql?(Allowed::TYPES_BOOLEAN)
+        new_value = BoolValue.true?(new_value) if @types.eql?(Allowed::TYPES_BOOLEAN)
         new_value = Integer(new_value) if @types.eql?(Allowed::TYPES_INTEGER)
         new_value = [new_value] if @types.eql?(Allowed::TYPES_STRING_ARRAY) && new_value.is_a?(String)
         # Setting a Hash to null set an empty hash
@@ -142,20 +173,7 @@ module Aspera
     # arguments options start with '-', others are commands
     # resolves on extended value syntax
     class Manager
-      BOOLEAN_SIMPLE = %i[no yes].freeze
       class << self
-        # @return `true` if value is a value for `true` in BOOLEAN_VALUES
-        def enum_to_bool(enum)
-          Aspera.assert_values(enum, BOOLEAN_VALUES){'boolean'}
-          return TRUE_VALUES.include?(enum)
-        end
-
-        # @return :yes ot :no
-        def enum_to_yes_no(enum)
-          Aspera.assert_values(enum, BOOLEAN_VALUES){'boolean'}
-          return TRUE_VALUES.include?(enum) ? BOOL_YES : BOOL_NO
-        end
-
         # Find shortened string value in allowed symbol list
         def get_from_list(short_value, descr, allowed_values)
           Aspera.assert_type(short_value, String)
@@ -165,7 +183,7 @@ module Aspera
           matching = allowed_values.select{ |i| i.to_s.start_with?(short_value)}
           Aspera.assert(!matching.empty?, multi_choice_assert_msg("unknown value for #{descr}: #{short_value}", allowed_values), type: BadArgument)
           Aspera.assert(matching.length.eql?(1), multi_choice_assert_msg("ambiguous shortcut for #{descr}: #{short_value}", matching), type: BadArgument)
-          return enum_to_bool(matching.first) if allowed_values.eql?(BOOLEAN_VALUES)
+          return BoolValue.true?(matching.first) if allowed_values.eql?(BoolValue::ALL)
           return matching.first
         end
 
@@ -279,11 +297,11 @@ module Aspera
         case option_attrs.types
         when Allowed::TYPES_ENUM, Allowed::TYPES_BOOLEAN
           # This option value must be a symbol (or array of symbols)
-          set_option(option_symbol, Manager.enum_to_bool(default), where: 'default') if option_attrs.values.eql?(BOOLEAN_VALUES) && !default.nil?
+          set_option(option_symbol, BoolValue.true?(default), where: 'default') if option_attrs.values.eql?(BoolValue::ALL) && !default.nil?
           value = get_option(option_symbol)
           help_values =
             if option_attrs.types.eql?(Allowed::TYPES_BOOLEAN)
-              highlight_current_in_list(BOOLEAN_SIMPLE, self.class.enum_to_yes_no(value))
+              highlight_current_in_list(BoolValue::SYMBOLS, BoolValue.to_sym(value))
             else
               highlight_current_in_list(option_attrs.values, value)
             end
@@ -312,7 +330,7 @@ module Aspera
 
       # @param descr       [String] description for help
       # @param mandatory   [Boolean] if true, raise error if option not set
-      # @param multiple    [Boolean] if true, return remaining arguments (Array)
+      # @param multiple    [Boolean] if true, return remaining arguments (Array) unil END
       # @param accept_list [Array, NilClass] list of allowed values (Symbol)
       # @param validation  [Class, Array, NilClass] Accepted value type(s) or list of Symbols
       # @param aliases     [Hash] map of aliases: key = alias, value = real value
@@ -327,10 +345,19 @@ module Aspera
         descr = "#{descr} (#{validation.join(', ')})" unless validation.nil? || validation.eql?(Allowed::TYPES_STRING)
         result =
           if !@unprocessed_cmd_line_arguments.empty?
-            how_many = multiple ? @unprocessed_cmd_line_arguments.length : 1
-            values = @unprocessed_cmd_line_arguments.shift(how_many)
+            if multiple
+              index = @unprocessed_cmd_line_arguments.index(SpecialValues::EOA)
+              if index.nil?
+                values = @unprocessed_cmd_line_arguments.shift(@unprocessed_cmd_line_arguments.length)
+              else
+                values = @unprocessed_cmd_line_arguments.shift(index)
+                @unprocessed_cmd_line_arguments.shift # remove EOA
+              end
+            else
+              values = [@unprocessed_cmd_line_arguments.shift]
+            end
             values = values.map{ |v| ExtendedValue.instance.evaluate(v, context: "argument: #{descr}", allowed: validation)}
-            # if expecting list and only one arg of type array : it is the list
+            # If expecting list and only one arg of type array : it is the list
             values = values.first if multiple && values.length.eql?(1) && values.first.is_a?(Array)
             if accept_list
               allowed_values = [].concat(accept_list)
@@ -636,12 +663,6 @@ module Aspera
           unprocessed_options.delete(k)
         end
       end
-      # boolean options are set to true/false from the following values
-      BOOL_YES = BOOLEAN_SIMPLE.last
-      BOOL_NO = BOOLEAN_SIMPLE.first
-      FALSE_VALUES = [BOOL_NO, false].freeze
-      TRUE_VALUES = [BOOL_YES, true].freeze
-      BOOLEAN_VALUES = (TRUE_VALUES + FALSE_VALUES).freeze
 
       # Option name separator on command line, e.g. in --option-blah, third "-"
       OPTION_SEP_LINE = '-'
@@ -655,7 +676,7 @@ module Aspera
       OPTIONS_STOP = '--'
       SOURCE_USER = 'cmdline' # cspell:disable-line
 
-      private_constant :BOOL_YES, :BOOL_NO, :FALSE_VALUES, :TRUE_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :OPTION_VALUE_SEPARATOR, :OPTION_PREFIX, :OPTIONS_STOP, :SOURCE_USER
+      private_constant :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :OPTION_VALUE_SEPARATOR, :OPTION_PREFIX, :OPTIONS_STOP, :SOURCE_USER
     end
   end
 end