aspera-cli 4.18.0 → 4.18.1

data/lib/aspera/api/node.rb

@@ -14,23 +14,28 @@ module Aspera
   module Api
     # Provides additional functions using node API with gen4 extensions (access keys)
     class Node < Aspera::Rest
-      # node api permissions
-      ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
-      HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       SCOPE_SEPARATOR = ':'
-      SCOPE_USER = 'user:all'
-      SCOPE_ADMIN = 'admin:all'
       SCOPE_NODE_PREFIX = 'node.'
       # prefix for ruby code for filter (deprecated)
       MATCH_EXEC_PREFIX = 'exec:'
       MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
-      PATH_SEPARATOR = '/'
       SIGNATURE_DELIMITER = '==SIGNATURE=='
       BEARER_TOKEN_VALIDITY_DEFAULT = 86400
-      BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
-      private_constant :MATCH_EXEC_PREFIX, :MATCH_TYPES,
-        :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT, :BEARER_TOKEN_SCOPE_DEFAULT,
-        :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX
+      # fields in @app_info
+      REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
+      # methods of @app_info[:api]
+      REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
+      private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_EXEC_PREFIX, :MATCH_TYPES,
+        :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
+        :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
+
+      # node api permissions
+      ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
+      HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
+      HEADER_X_TOTAL_COUNT = 'X-Total-Count'
+      SCOPE_USER = 'user:all'
+      SCOPE_ADMIN = 'admin:all'
+      PATH_SEPARATOR = '/'
 
       # register node special token decoder
       OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
@@ -59,7 +64,7 @@ module Aspera
         end
 
         def file_matcher_from_argument(options)
-          return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
+          return file_matcher(options.get_next_argument('filter', validation: MATCH_TYPES, mandatory: false))
         end
 
         # node API scopes
@@ -86,7 +91,7 @@ module Aspera
           # manage convenience parameters
           expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
           payload.delete('_validity')
-          scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
+          scope = payload['_scope'] || SCOPE_USER
           payload.delete('_scope')
           payload['scope'] ||= token_scope(access_key, scope)
           payload['auth_type'] ||= 'access_key'
@@ -117,19 +122,13 @@ module Aspera
         end
       end
 
-      # fields in @app_info
-      REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
-      # methods of @app_info[:api]
-      REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
-      private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
-
       attr_reader :app_info
 
+      # @param app_info  [Hash,NilClass]   Special processing for AoC
+      # @param add_tspec [Hash,NilClass]   Additional transfer spec
       # @param base_url  [String]          Rest parameters
       # @param auth      [String,NilClass] Rest parameters
       # @param headers   [String,NilClass] Rest parameters
-      # @param app_info  [Hash,NilClass]   Special processing for AoC
-      # @param add_tspec [Hash,NilClass]   Additional transfer spec
       def initialize(app_info: nil, add_tspec: nil, **rest_args)
         # init Rest
         super(**rest_args)
@@ -162,25 +161,42 @@ module Aspera
             workspace_id: @app_info[:workspace_id],
             workspace_name: @app_info[:workspace_name])
         end
-        Log.log.warn{"cannot resolve link with node id #{node_id}"}
+        Log.log.warn{"Cannot resolve link with node id #{node_id}, no resolver"}
         return nil
       end
 
+      # Check if a link entry in folder has target information
+      # @param entry [Hash] entry in folder
+      # @return [Boolean] true if target information is available
+      def entry_has_link_information(entry)
+        # if target information is missing in folder, try to get it on entry
+        if entry['target_node_id'].nil? || entry['target_id'].nil?
+          link_entry = read("files/#{entry['id']}")[:data]
+          entry['target_node_id'] = link_entry['target_node_id']
+          entry['target_id'] = link_entry['target_id']
+        end
+        return true unless entry['target_node_id'].nil? || entry['target_id'].nil?
+        Log.log.warn{"Missing target information for link: #{entry['name']}"}
+        return false
+      end
+
       # Recursively browse in a folder (with non-recursive method)
       # sub folders are processed if the processing method returns true
+      # links are processed on the respective node
       # @param state [Object] state object sent to processing method
       # @param top_file_id [String] file id to start at (default = access key root file id)
       # @param top_file_path [String] path of top folder (default = /)
       # @param block [Proc] processing method, arguments: entry, path, state
-      def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
+      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
         Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
-        Aspera.assert(block){'Missing block'}
+        Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
         # start at top folder
         folders_to_explore = [{id: top_file_id, path: top_file_path}]
         Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
         until folders_to_explore.empty?
+          # consume first in job list
           current_item = folders_to_explore.shift
-          Log.log.debug{"searching #{current_item[:path]}".bg_green}
+          Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
           # get folder content
           folder_contents =
             begin
@@ -192,19 +208,21 @@ module Aspera
           Log.log.debug{Log.dump(:folder_contents, folder_contents)}
           folder_contents.each do |entry|
             relative_path = File.join(current_item[:path], entry['name'])
-            Log.log.debug{"process_folder_tree checking #{relative_path}"}
-            # continue only if method returns true
-            next unless yield(entry, relative_path, state)
+            Log.log.debug{"process_folder_tree: checking #{relative_path}"}
+            # call block, continue only if method returns true
+            next unless send(method_sym, entry, relative_path, state)
             # entry type is file, folder or link
             case entry['type']
             when 'folder'
               folders_to_explore.push({id: entry['id'], path: relative_path})
             when 'link'
-              node_id_to_node(entry['target_node_id'])&.process_folder_tree(
-                state:         state,
-                top_file_id:   entry['target_id'],
-                top_file_path: relative_path,
-                &block)
+              if entry_has_link_information(entry)
+                node_id_to_node(entry['target_node_id'])&.process_folder_tree(
+                  method_sym:    method_sym,
+                  state:         state,
+                  top_file_id:   entry['target_id'],
+                  top_file_path: relative_path)
+              end
             end
           end
         end
@@ -216,60 +234,23 @@ module Aspera
       # @return [Hash] {.api,.file_id}
       def resolve_api_fid(top_file_id, path)
         Aspera.assert_type(top_file_id, String)
+        Aspera.assert_type(path, String)
+        # if last element is a link and followed by "/", we list the content of that folder, else we return the link
         process_last_link = path.end_with?(PATH_SEPARATOR)
+        # keep only non-empty elements
         path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
         return {api: self, file_id: top_file_id} if path_elements.empty?
-        resolve_state = {path: path_elements, result: nil}
-        process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
-          # this block is called recursively for each entry in folder
-          # stop digging here if not in right path
-          next false unless entry['name'].eql?(state[:path].first)
-          # ok it matches, so we remove the match
-          state[:path].shift
-          case entry['type']
-          when 'file'
-            # file must be terminal
-            raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
-            # it's terminal, we found it
-            state[:result] = {api: self, file_id: entry['id']}
-            next false
-          when 'folder'
-            if state[:path].empty?
-              # we found it
-              state[:result] = {api: self, file_id: entry['id']}
-              next false
-            end
-          when 'link'
-            if state[:path].empty?
-              if process_last_link
-                # we found it
-                other_node = node_id_to_node(entry['target_node_id'])
-                raise 'cannot resolve link' if other_node.nil?
-                state[:result] = {api: other_node, file_id: entry['target_id']}
-              else
-                # we found it but we do not process the link
-                state[:result] = {api: self, file_id: entry['id']}
-              end
-              next false
-            end
-          else
-            Log.log.warn{"Unknown element type: #{entry['type']}"}
-          end
-          # continue to dig folder
-          next true
-        end
+        resolve_state = {path: path_elements, result: nil, process_last_link: process_last_link}
+        process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
         raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
+        Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result][:api].base_url} #{resolve_state[:result][:file_id]}"}
         return resolve_state[:result]
       end
 
       def find_files(top_file_id, test_block)
         Log.log.debug{"find_files: file id=#{top_file_id}"}
         find_state = {found: [], test_block: test_block}
-        process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
-          state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
-          # test all files deeply
-          true
-        end
+        process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
         return find_state[:found]
       end
 
@@ -306,7 +287,7 @@ module Aspera
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
           # TODO: token_generation_lambda = lambda{|do_refresh|oauth.token(refresh: do_refresh)}
           # get bearer token, possibly use cache
-          ak_token = oauth.token(refresh: false)
+          ak_token = oauth.token
         else Aspera.error_unexpected_value(auth_params[:type])
         end
         transfer_spec = {
@@ -352,6 +333,58 @@ module Aspera
           unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
         return transfer_spec
       end
+
+      private
+
+      def process_api_fid(entry, path, state)
+        # this block is called recursively for each entry in folder
+        # stop digging here if not in right path
+        return false unless entry['name'].eql?(state[:path].first)
+        # ok it matches, so we remove the match, and continue digging
+        state[:path].shift
+        path_fully_consumed = state[:path].empty?
+        case entry['type']
+        when 'file'
+          # file must be terminal
+          raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless path_fully_consumed
+          # it's terminal, we found it
+          Log.log.debug{"resolve_api_fid: found #{path} -> #{entry['id']}"}
+          state[:result] = {api: self, file_id: entry['id']}
+          return false
+        when 'folder'
+          if path_fully_consumed
+            # we found it
+            state[:result] = {api: self, file_id: entry['id']}
+            return false
+          end
+        when 'link'
+          if path_fully_consumed
+            if state[:process_last_link]
+              # we found it
+              other_node = nil
+              if entry_has_link_information(entry)
+                other_node = node_id_to_node(entry['target_node_id'])
+              end
+              raise 'Cannot resolve link' if other_node.nil?
+              state[:result] = {api: other_node, file_id: entry['target_id']}
+            else
+              # we found it but we do not process the link
+              state[:result] = {api: self, file_id: entry['id']}
+            end
+            return false
+          end
+        else
+          Log.log.warn{"Unknown element type: #{entry['type']}"}
+        end
+        # continue to dig folder
+        return true
+      end
+
+      def process_find_files(entry, _path, state)
+        state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
+        # test all files deeply
+        return true
+      end
     end
   end
 end

data/lib/aspera/ascp/products.rb

@@ -44,7 +44,7 @@ module Aspera
               app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
               log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
             }]
-            when Aspera::Environment::OS_X then [{
+            when Aspera::Environment::OS_MACOS then [{
               expected: CONNECT,
               app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
               log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),

data/lib/aspera/cli/extended_value.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# cspell:ignore csvt jsonpp
+# cspell:ignore csvt jsonpp stdbin
 require 'aspera/uri_reader'
 require 'aspera/environment'
 require 'aspera/log'
@@ -17,11 +17,6 @@ module Aspera
     class ExtendedValue
       include Singleton
 
-      # special values
-      INIT = 'INIT'
-      ALL = 'ALL'
-      DEF = 'DEF'
-
       MARKER_START = '@'
       MARKER_END = ':'
       MARKER_IN_END = '@'
@@ -74,10 +69,22 @@ module Aspera
           zlib:   lambda{|v|Zlib::Inflate.inflate(v)},
           extend: lambda{|v|ExtendedValue.instance.evaluate_all(v)}
         }
+        @default_decoder = nil
+      end
+
+      # Regex to match an extended value
+      def handler_regex_string
+        "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
       end
 
       public
 
+      def default_decoder=(value)
+        Log.log.debug{"setting default decoder to #{value} (#{value.class})"}
+        Aspera.assert(value.nil? || @handlers.key?(value))
+        @default_decoder = value
+      end
+
       def modifiers; @handlers.keys; end
 
       # add a new handler
@@ -87,16 +94,13 @@ module Aspera
         @handlers[name] = method
       end
 
-      # Regex to match an extended value
-      def ext_re
-        "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
-      end
-
-      # parse an option value if it is a String using supported extended value modifiers
+      # parse an string value to extended value, if it is a String using supported extended value modifiers
       # other value types are returned as is
+      # @param value [String] the value to parse
+      # @param expect [Class,Array] one or a list of expected types
       def evaluate(value)
         return value unless value.is_a?(String)
-        regex = Regexp.new("^#{ext_re}(.*)$", Regexp::MULTILINE)
+        regex = Regexp.new("^#{handler_regex_string}(.*)$", Regexp::MULTILINE)
         # first determine decoders, in reversed order
         handlers_reversed = []
         while (m = value.match(regex))
@@ -113,9 +117,18 @@ module Aspera
         return value
       end
 
+      # parse string value as extended value
+      # use default decoder if none is specified
+      def evaluate_with_default(value)
+        if value.is_a?(String) && value.match(/^#{handler_regex_string}.*$/).nil? && !@default_decoder.nil?
+          value = [MARKER_START, @default_decoder, MARKER_END, value].join
+        end
+        return evaluate(value)
+      end
+
       # find inner extended values
       def evaluate_all(value)
-        regex = Regexp.new("^(.*)#{ext_re}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
+        regex = Regexp.new("^(.*)#{handler_regex_string}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
         while (m = value.match(regex))
           sub_value = "@#{m[2]}:#{m[3]}"
           Log.log.debug{"evaluating #{sub_value}"}

data/lib/aspera/cli/formatter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 # cspell:ignore jsonpp
+require 'aspera/cli/special_values'
 require 'aspera/preview/terminal'
 require 'aspera/secret_hider'
 require 'aspera/environment'
@@ -110,7 +111,7 @@ module Aspera
       class << self
         def all_but(list)
           list = [list] unless list.is_a?(Array)
-          return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(ExtendedValue::ALL)
+          return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(SpecialValues::ALL)
         end
 
         def tick(yes)
@@ -207,9 +208,9 @@ module Aspera
         options.declare(:output, 'Destination for results', types: String, handler: {o: self, m: :option_handler})
         options.declare(:display, 'Output only some information', values: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :info)
         options.declare(
-          :fields, "Comma separated list of: fields, or #{ExtendedValue::ALL}, or #{ExtendedValue::DEF}", handler: {o: self, m: :option_handler},
+          :fields, "Comma separated list of: fields, or #{SpecialValues::ALL}, or #{SpecialValues::DEF}", handler: {o: self, m: :option_handler},
           types: [String, Array, Regexp, Proc],
-          default: ExtendedValue::DEF)
+          default: SpecialValues::DEF)
         options.declare(:select, 'Select only some items in lists: column, value', types: [Hash, Proc], handler: {o: self, m: :option_handler})
         options.declare(:table_style, 'Table display style', types: [Hash], handler: {o: self, m: :option_handler}, default: default_table_style)
         options.declare(:flat_hash, 'Display deep values as additional keys', values: :bool, handler: {o: self, m: :option_handler}, default: true)
@@ -256,7 +257,7 @@ module Aspera
         # the requested list of fields, but if can contain special values
         request =
           case @options[:fields]
-          # when NilClass then [ExtendedValue::DEF]
+          # when NilClass then [SpecialValues::DEF]
           when String then @options[:fields].split(',')
           when Array then @options[:fields]
           when Regexp then return all_fields(data).select{|i|i.match(@options[:fields])}
@@ -272,10 +273,10 @@ module Aspera
             item = item[1..-1]
           end
           case item
-          when ExtendedValue::ALL
+          when SpecialValues::ALL
             # get the list of all column names used in all lines, not just first one, as all lines may have different columns
             request.unshift(*all_fields(data))
-          when ExtendedValue::DEF
+          when SpecialValues::DEF
             default = all_fields(data).select{|i|default.call(i)} if default.is_a?(Proc)
             default = all_fields(data) if default.nil?
             request.unshift(*default)
@@ -293,11 +294,11 @@ module Aspera
       # filter the list of items on the fields option
       def filter_list_on_fields(data)
         # by default, keep all data intact
-        return data if @options[:fields].eql?(ExtendedValue::DEF) && @options[:select].nil?
+        return data if @options[:fields].eql?(SpecialValues::DEF) && @options[:select].nil?
         Aspera.assert_type(data, Array){'Filtering fields or select requires result is an Array of Hash'}
         Aspera.assert(data.all?(Hash)){'Filtering fields or select requires result is an Array of Hash'}
         filter_columns_on_select(data)
-        return data if @options[:fields].eql?(ExtendedValue::DEF)
+        return data if @options[:fields].eql?(SpecialValues::DEF)
         selected_fields = compute_fields(data, @options[:fields])
         return data.map{|i|i[selected_fields.first]} if selected_fields.length == 1
         return data.map{|i|i.select{|k, _|selected_fields.include?(k)}}
@@ -358,8 +359,8 @@ module Aspera
           raise URI::InvalidURIError, 'not uri' if !(blob =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/)
           # it's a url
           url = blob
-          unless OpenApplication.instance.url_method.eql?(:text)
-            OpenApplication.instance.uri(url)
+          unless Environment.instance.url_method.eql?(:text)
+            Environment.instance.open_uri(url)
             return ''
           end
           # remote_image = Rest.new(base_url: url).read('')

data/lib/aspera/cli/main.rb

@@ -93,11 +93,11 @@ module Aspera
         @plug_init[:only_manual] = false
         # create formatter, in case there is an exception, it is used to display.
         @plug_init[:formatter] = Formatter.new
-        @plug_init[:options] = Manager.new(PROGRAM_NAME)
-        # give command line arguments to option manager
-        options.parse_command_line(@argv)
+        # create command line manager with arguments
+        @plug_init[:options] = Manager.new(PROGRAM_NAME, @argv)
         # formatter adds options
-        formatter.declare_options(options)
+        @plug_init[:formatter].declare_options(options)
+        ExtendedValue.instance.default_decoder = options.get_option(:struct_parser)
         # compare $0 with expected name
         current_prog_name = File.basename($PROGRAM_NAME)
         formatter.display_message(
@@ -162,9 +162,9 @@ module Aspera
         options.declare(:version, 'Display version', values: :none, short: 'v') { formatter.display_message(:data, Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
         options.declare(
           :ui, 'Method to start browser',
-          values: OpenApplication::USER_INTERFACES,
-          handler: {o: OpenApplication.instance, m: :url_method},
-          default: OpenApplication.default_gui_mode)
+          values: Environment::USER_INTERFACES,
+          handler: {o: Environment.instance, m: :url_method},
+          default: Environment.default_gui_mode)
         options.declare(:log_level, 'Log level', values: Log.levels, handler: {o: Log.instance, m: :level})
         options.declare(:logger, 'Logging method', values: Log::LOG_TYPES, handler: {o: Log.instance, m: :logger_type})
         options.declare(:lock_port, 'Prevent dual execution of a command, e.g. in cron', coerce: Integer, types: Integer)
@@ -189,7 +189,7 @@ module Aspera
       end
 
       def generate_bash_completion
-        if options.get_next_argument('', expected: :multiple, mandatory: false).nil?
+        if options.get_next_argument('', multiple: true, mandatory: false).nil?
           PluginFactory.instance.plugin_list.each{|p|puts p}
         else
           Log.log.warn('only first level completion so far')
@@ -197,11 +197,11 @@ module Aspera
         Process.exit(0)
       end
 
-      def exit_with_usage(all_plugins)
-        Log.log.debug('exit_with_usage'.bg_red)
+      def exit_with_usage(include_all_plugins)
+        Log.log.debug{"exit_with_usage(#{include_all_plugins})".bg_red}
         # display main plugin options
         formatter.display_message(:error, options.parser)
-        if all_plugins
+        if include_all_plugins
           # list plugins that have a "require" field, i.e. all but main plugin
           PluginFactory.instance.plugin_list.each do |plugin_name_sym|
             next if plugin_name_sym.eql?(COMMAND_CONFIG)