aspera-cli 4.24.1 → 4.25.0.pre

data/lib/aspera/api/node.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'aspera/cli/error'
 require 'aspera/transfer/spec'
 require 'aspera/rest'
 require 'aspera/oauth'
@@ -17,14 +16,18 @@ module Aspera
   module Api
     # Provides additional functions using node API with gen4 extensions (access keys)
     class Node < Aspera::Rest
+      # Separator between node.AK and user:all
       SCOPE_SEPARATOR = ':'
       SCOPE_NODE_PREFIX = 'node.'
+      # Accepted types in `file_matcher`
       MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
+      # Delimiter in decoded node token
       SIGNATURE_DELIMITER = '==SIGNATURE=='
+      # Default validity when generating a bearer token "manually"
       BEARER_TOKEN_VALIDITY_DEFAULT = 86400
-      # fields in @app_info
+      # Fields in @app_info
       REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
-      # methods of @app_info[:api]
+      # Methods of @app_info[:api]
       REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
       private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_TYPES,
         :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
@@ -32,30 +35,40 @@ module Aspera
 
       # Node API permissions
       ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
+      # Special HTTP Headers
       HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       HEADER_X_TOTAL_COUNT = 'X-Total-Count'
       HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
       HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
+      # Node sub-scopes
       SCOPE_USER = 'user:all'
       SCOPE_ADMIN = 'admin:all'
       # / in cloud
       PATH_SEPARATOR = '/'
 
-      # register node special token decoder
+      # Register node special token decoder
       OAuth::Factory.instance.register_decoder(lambda{ |token| Node.decode_bearer_token(token)})
 
-      # class instance variable, access with accessors on class
+      # Class instance variable, access with accessors on class
       @use_standard_ports = true
       @use_node_cache = true
 
       class << self
-        # set to false to read transfer parameters from download_setup
+        # Set to false to read transfer parameters from download_setup
         attr_accessor :use_standard_ports
-        # set to false to bypass cache in redis
+        # Set to false to bypass cache in redis
         attr_accessor :use_node_cache
         attr_reader :use_dynamic_key
 
-        # set private key to be used
+        # Adds cache control header, as globally specified to read request
+        # Use like this: read(...,**cache_control)
+        def cache_control
+          headers = {'Accept' => Rest::MIME_JSON}
+          headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
+          {headers: headers}
+        end
+
+        # Set private key to be used
         # @param pem_content [String] PEM encoded private key
         def use_dynamic_key=(pem_content)
           Aspera.assert_type(pem_content, String)
@@ -67,7 +80,7 @@ module Aspera
         def add_public_key(h)
           if @dynamic_key
             ssh_key = Net::SSH::Buffer.from(:key, @dynamic_key)
-            # get pub key in OpenSSH public key format (authorized_keys)
+            # Get pub key in OpenSSH public key format (authorized_keys)
             h['public_keys'] = [
               ssh_key.read_string,
               Base64.strict_encode64(ssh_key.to_s)
@@ -93,14 +106,16 @@ module Aspera
           when String
             return ->(f){File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
           when NilClass then return ->(_){true}
-          else Aspera.error_unexpected_value(match_expression.class.name, type: Cli::BadArgument)
+          else Aspera.error_unexpected_value(match_expression.class.name, type: ParameterError)
           end
         end
 
+        # @return [Proc] lambda from provided CLI options
         def file_matcher_from_argument(options)
           return file_matcher(options.get_next_argument('filter', validation: MATCH_TYPES, mandatory: false))
         end
 
+        # Split path into folder + filename
         # @return [Array] containing folder + inside folder/file
         def split_folder(path)
           folder = path.split(PATH_SEPARATOR)
@@ -108,11 +123,14 @@ module Aspera
           [folder.join(PATH_SEPARATOR), inside]
         end
 
-        # node API scopes
+        # Node API scopes
+        # @return [String] node scope
         def token_scope(access_key, scope)
           return [SCOPE_NODE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
         end
 
+        # Decode node scope into access key and scope
+        # @return [Hash]
         def decode_scope(scope)
           items = scope.split(SCOPE_SEPARATOR, 2)
           Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
@@ -130,7 +148,7 @@ module Aspera
           Aspera.assert_type(payload['user_id'], String)
           Aspera.assert(!payload['user_id'].empty?)
           Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
-          # manage convenience parameters
+          # Manage convenience parameters
           expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
           payload.delete('_validity')
           scope = payload['_scope'] || SCOPE_USER
@@ -152,8 +170,9 @@ module Aspera
           return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
         end
 
+        # @return [Hash] Headers to call node API with access key and auth
         def bearer_headers(bearer_auth, access_key: nil)
-          # if username is not provided, use the access key from the token
+          # If username is not provided, use the access key from the token
           if access_key.nil?
             access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_token(bearer_auth))['scope'])[:access_key]
             Aspera.assert(!access_key.nil?)
@@ -167,17 +186,17 @@ module Aspera
 
       attr_reader :app_info
 
-      # @param app_info  [Hash,NilClass]   Special processing for AoC
+      # @param app_info  [Hash,NilClass]   App information, typically AoC
       # @param add_tspec [Hash,NilClass]   Additional transfer spec
       # @param base_url  [String]          Rest parameters
       # @param auth      [String,NilClass] Rest parameters
       # @param headers   [String,NilClass] Rest parameters
       def initialize(app_info: nil, add_tspec: nil, **rest_args)
-        # init Rest
+        # Init Rest
         super(**rest_args)
         @dynamic_key = nil
         @app_info = app_info
-        # this is added to transfer spec, for instance to add tags (COS)
+        # This is added to transfer spec, for instance to add tags (COS)
         @add_tspec = add_tspec
         @std_t_spec_cache = nil
         if !@app_info.nil?
@@ -190,25 +209,15 @@ module Aspera
         end
       end
 
-      # Call node API, possibly adding cache control header, as globally specified
-      def read_with_cache(subpath, query = nil)
-        headers = {'Accept' => Rest::MIME_JSON}
-        headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
-        return call(
-          operation: 'GET',
-          subpath:   subpath,
-          headers:   headers,
-          query:     query
-        )[:data]
-      end
-
-      # update transfer spec with special additional tags
+      # Update transfer spec with special additional tags
+      # @param tspec [Hash] Transfer spec to be modified
+      # @return [Hash] initial modified tspec
       def add_tspec_info(tspec)
         tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
         return tspec
       end
 
-      # @returns [Node] a Node or nil
+      # @returns [Node] a Node Api object or nil if no App defined
       def node_id_to_node(node_id)
         if !@app_info.nil?
           return self if node_id.eql?(@app_info[:node_info]['id'])
@@ -226,7 +235,7 @@ module Aspera
       # @param entry [Hash] entry in folder
       # @return [Boolean] true if target information is available
       def entry_has_link_information(entry)
-        # if target information is missing in folder, try to get it on entry
+        # If target information is missing in folder, try to get it on entry
         if entry['target_node_id'].nil? || entry['target_id'].nil?
           link_entry = read("files/#{entry['id']}")
           entry['target_node_id'] = link_entry['target_node_id']
@@ -238,27 +247,28 @@ module Aspera
       end
 
       # Recursively browse in a folder (with non-recursive method)
-      # sub folders are processed if the processing method returns true
-      # links are processed on the respective node
+      # Entries of folders are processed if the processing method returns true
+      # Links are processed on the respective node
       # @param method_sym [Symbol] processing method, arguments: entry, path, state
       # @param state [Object] state object sent to processing method
       # @param top_file_id [String] file id to start at (default = access key root file id)
       # @param top_file_path [String] path of top folder (default = /)
-      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
+      # @para query [Hash, nil] optional query for `read`
+      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
         Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
         Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
-        # start at top folder
+        # Start at top folder
         folders_to_explore = [{id: top_file_id, path: top_file_path}]
         Log.dump(:folders_to_explore, folders_to_explore)
         until folders_to_explore.empty?
-          # consume first in job list
+          # Consume first in job list
           current_item = folders_to_explore.shift
           Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
-          # get folder content
+          # Get folder content
           folder_contents =
             begin
               # TODO: use header
-              read_with_cache("files/#{current_item[:id]}/files")
+              read("files/#{current_item[:id]}/files", query, **self.class.cache_control)
             rescue StandardError => e
               Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
               []
@@ -271,9 +281,9 @@ module Aspera
             end
             current_path = File.join(current_item[:path], entry['name'])
             Log.log.debug{"process_folder_tree: checking #{current_path}"}
-            # call block, continue only if method returns true
+            # Call block, continue only if method returns true
             next unless send(method_sym, entry, current_path, state)
-            # entry type is file, folder or link
+            # Entry type is file, folder or link
             case entry['type']
             when 'folder'
               folders_to_explore.push({id: entry['id'], path: current_path})
@@ -303,9 +313,9 @@ module Aspera
         process_last_link ||= path.end_with?(PATH_SEPARATOR)
         path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
         return {api: self, file_id: top_file_id} if path_elements.empty?
-        resolve_state = {path: path_elements, result: nil, process_last_link: process_last_link}
+        resolve_state = {path: path_elements, consumed: [], result: nil, process_last_link: process_last_link}
         process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
-        raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
+        raise ParameterError, "Entry not found: #{resolve_state[:path].first} in /#{resolve_state[:consumed].join(PATH_SEPARATOR)}" if resolve_state[:result].nil?
         Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result][:api].base_url} #{resolve_state[:result][:file_id]}"}
         return resolve_state[:result]
       end
@@ -338,14 +348,14 @@ module Aspera
           source_paths =
             case file_info['type']
             when 'file'
-              # if the single source is a file, we need to split into folder path and filename
+              # If the single source is a file, we need to split into folder path and filename
               src_dir_elements = source_folder.split(Api::Node::PATH_SEPARATOR)
               filename = src_dir_elements.pop
               apifid = resolve_api_fid(top_file_id, src_dir_elements.join(Api::Node::PATH_SEPARATOR), true)
-              # filename is the last one, source folder is what remains
+              # Filename is the last one, source folder is what remains
               [{'source' => filename}]
             when 'link', 'folder'
-              # single source is 'folder' or 'link'
+              # Single source is 'folder' or 'link'
               # TODO: add this ? , 'destination'=>file_info['name']
               [{'source' => '.'}]
             else Aspera.error_unexpected_value(file_info['type']){'source type'}
@@ -354,6 +364,9 @@ module Aspera
         [apifid, source_paths]
       end
 
+      # Recursively find files matching lambda
+      # @param top_file_id [String] Search root
+      # @param test_lambda [Proc] Test function
       def find_files(top_file_id, test_lambda)
         Log.log.debug{"find_files: file id=#{top_file_id}"}
         find_state = {found: [], test_lambda: test_lambda}
@@ -361,36 +374,37 @@ module Aspera
         return find_state[:found]
       end
 
-      def list_files(top_file_id)
+      # Recursively list all files and folders
+      def list_files(top_file_id, query: nil)
         find_state = {found: []}
-        process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id)
+        process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id, query: query)
         return find_state[:found]
       end
 
+      # Generate a refreshed auth token
       def refreshed_transfer_token
         return oauth.authorization(refresh: true)
       end
 
-      # @return part of transfer spec with transport parameters only
+      # Get a base download transfer spec (gen3)
+      # @return [Hash] Base transfer spec
+      def base_spec
+        create(
+          'files/download_setup',
+          {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
+        )['transfer_specs'].first['transfer_spec']
+      end
+
+      # Get generic part of transfer spec with transport parameters only
+      # @return [Hash] Base transfer spec
       def transport_params
-        if @std_t_spec_cache.nil?
-          # retrieve values from API (and keep a copy/cache)
-          full_spec = create(
-            'files/download_setup',
-            {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
-          )['transfer_specs'].first['transfer_spec']
-          # set available fields
-          @std_t_spec_cache = Transfer::Spec::TRANSPORT_FIELDS.each_with_object({}) do |i, h|
-            h[i] = full_spec[i] if full_spec.key?(i)
-          end
-        end
-        return @std_t_spec_cache
+        @std_t_spec_cache ||= base_spec.slice(*Transfer::Spec::TRANSPORT_FIELDS).freeze
       end
 
       # Create transfer spec for gen4
-      # @param file_id destination or source folder (id)
-      # @param direction one of Transfer::Spec::DIRECTION_SEND, Transfer::Spec::DIRECTION_RECEIVE
-      # @param ts_merge additional transfer spec to merge
+      # @param file_id   [String]   Destination or source folder (id)
+      # @param direction [Symbol]   One of Transfer::Spec::DIRECTION_SEND, Transfer::Spec::DIRECTION_RECEIVE
+      # @param ts_merge  [Hash,nil] Additional transfer spec to merge
       def transfer_spec_gen4(file_id, direction, ts_merge = nil)
         ak_name = nil
         ak_token = nil
@@ -402,7 +416,7 @@ module Aspera
         when :oauth2
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
           # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
-          # get bearer token, possibly use cache
+          # Get bearer token, possibly use cache
           ak_token = oauth.authorization
         when :none
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
@@ -417,92 +431,91 @@ module Aspera
               'node' => {
                 'access_key' => ak_name,
                 'file_id'    => file_id
-              } # node
-            } # aspera
-          } # tags
+              }
+            }
+          }
         }
-        # add specials tags (cos)
+        # Add specials tags (cos)
         add_tspec_info(transfer_spec)
         transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
-        # add application specific tags (AoC)
-        app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
-        # add remote host info
+        # Add application specific tags (AoC)
+        @app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: @app_info) unless @app_info.nil?
+        # Add remote host info
         if self.class.use_standard_ports
-          # get default TCP/UDP ports and transfer user
+          # Get default TCP/UDP ports and transfer user
           transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
-          # by default: same address as node API
+          # By default: same address as node API
           transfer_spec['remote_host'] = URI.parse(base_url).host
           # AoC allows specification of other url
           transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url'] if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
           info = read('info')
-          # get the transfer user from info on access key
+          # Get the transfer user from info on access key
           transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
-          # get settings from name.value array to hash key.value
+          # Get settings from name.value array to hash key.value
           settings = info['settings']&.each_with_object({}){ |i, h| h[i['name']] = i['value']}
-          # check WSS ports
+          # Check WSS ports
           Transfer::Spec::WSS_FIELDS.each do |i|
             transfer_spec[i] = settings[i] if settings.key?(i)
           end if settings.is_a?(Hash)
         else
           transfer_spec.merge!(transport_params)
         end
-        Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
-          unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
+        Aspera.assert_values(transfer_spec['remote_user'], Transfer::Spec::ACCESS_KEY_TRANSFER_USER, type: :warn){'transfer user'}
         return transfer_spec
       end
 
       private
 
-      # method called in loop for each entry for `resolve_api_fid`
+      # Method called in loop for each entry for `resolve_api_fid`
+      # @return `true` to continue digging, `false` to stop processing: set state[:result] if found
       def process_api_fid(entry, path, state)
-        # stop digging here if not in right path
+        # Stop digging here if not in right path
         return false unless entry['name'].eql?(state[:path].first)
-        # ok it matches, so we remove the match, and continue digging
-        state[:path].shift
+        # Ok it matches, so we remove the match, and continue digging
+        state[:consumed].push(state[:path].shift)
         path_fully_consumed = state[:path].empty?
         case entry['type']
         when 'file'
-          # file must be terminal
+          # File must be terminal
           raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless path_fully_consumed
-          # it's terminal, we found it
-          Log.log.debug{"resolve_api_fid: found #{path} -> #{entry['id']}"}
+          # It's terminal, we found it
+          Log.log.debug{"process_api_fid: found #{path} -> #{entry['id']}"}
           state[:result] = {api: self, file_id: entry['id']}
           return false
         when 'folder'
           if path_fully_consumed
-            # we found it
+            # We found it
             state[:result] = {api: self, file_id: entry['id']}
             return false
           end
         when 'link'
           if path_fully_consumed
             if state[:process_last_link]
-              # we found it
+              # We found it
               other_node = nil
               other_node = node_id_to_node(entry['target_node_id']) if entry_has_link_information(entry)
               raise Error, 'Cannot resolve link' if other_node.nil?
               state[:result] = {api: other_node, file_id: entry['target_id']}
             else
-              # we found it but we do not process the link
+              # We found it but we do not process the link
               state[:result] = {api: self, file_id: entry['id']}
             end
             return false
           end
-        else
-          Log.log.warn{"Unknown element type: #{entry['type']}"}
+        else Aspera.error_unexpected_value(entry['type'], type: :warn){'folder entry type'}
         end
-        # continue to dig folder
+        # Continue to dig folder
         return true
       end
 
-      # method called in loop for each entry for `find_files`
+      # Method called in loop for each entry for `find_files`
       def process_find_files(entry, path, state)
         state[:found].push(entry.merge({'path' => path})) if state[:test_lambda].call(entry)
-        # test all files deeply
+        # Test all files deeply
         return true
       end
 
-      # method called in loop for each entry for `list_files`
+      # Method called in loop for each entry for `list_files`
       def process_list_files(entry, path, state)
         state[:found].push(entry.merge({'path' => path}))
         return false

data/lib/aspera/ascmd.rb

@@ -94,8 +94,7 @@ module Aspera
       arguments = [] if arguments.nil?
       Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
       Aspera.assert_type(action_sym, Symbol)
-      Aspera.assert_type(arguments, Array)
-      Aspera.assert(arguments.all?(String), 'arguments must be strings')
+      Aspera.assert_array_all(arguments, String){'arguments'}
       remote_cmd = 'ascmd'
       # lines of commands (String's)
       command_lines = []