aspera-cli 4.24.2 → 4.25.0.pre2

data/lib/aspera/cli/plugins/node.rb

@@ -21,6 +21,30 @@ module Aspera
       class Node < BasicAuth
         include SyncActions
 
+        # Processing of paths in arguments and results
+        # Used only by Faspex4 to browse packages
+        class NodePathPrefix
+          def initialize(path)
+            @root = path
+          end
+
+          # get next path argument from command line, and add prefix
+          def add_to_path(path_arg)
+            File.join(@root, path_arg)
+          end
+
+          # get remaining path arguments from command line, and add prefix
+          def add_to_paths!(path_args)
+            path_args.map!{ |p| add_to_path(p)}
+          end
+
+          def remove_in_object_list!(obj_list)
+            obj_list.each do |item|
+              item['path'] = item['path'][@root.length..-1] if item['path'].start_with?(@root)
+            end
+          end
+        end
+
         class << self
           # directory: node, container: shares
           FOLDER_TYPES = %w[directory container].freeze
@@ -30,6 +54,7 @@ module Aspera
             'HSTS Node API'
           end
 
+          # @return [Hash,NilClass]
           def detect(address_or_url)
             urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
               [address_or_url]
@@ -45,12 +70,12 @@ module Aspera
               next unless base_url.match?('https?://')
               api = Rest.new(base_url: base_url)
               test_endpoint = 'ping'
-              result = api.call(operation: 'GET', subpath: test_endpoint)
-              next unless result[:http].body.eql?('')
+              http = api.read(test_endpoint, ret: :resp)
+              next unless http.body.eql?('')
               # also remove "/"
               url_end = -2 - test_endpoint.length
               return {
-                url:     result[:http].uri.to_s[0..url_end],
+                url:     http.uri.to_s[0..url_end],
                 version: 'requires authentication'
               }
             rescue StandardError => e
@@ -68,11 +93,11 @@ module Aspera
             options.declare(:validator, 'Identifier of validator (optional for central)')
             options.declare(:asperabrowserurl, 'URL for simple aspera web ui', default: 'https://asperabrowser.mybluemix.net')
             options.declare(
-              :default_ports, 'Gen4: Use standard FASP ports (true) or get from node API (false)', values: :bool, default: :yes,
+              :default_ports, 'Gen4: Use standard FASP ports (true) or get from node API (false)', allowed: Allowed::TYPES_BOOLEAN, default: true,
               handler: {o: Api::Node, m: :use_standard_ports}
             )
             options.declare(
-              :node_cache, 'Gen4: Set to no to force actual file system read', values: :bool,
+              :node_cache, 'Gen4: Set to no to force actual file system read', allowed: Allowed::TYPES_BOOLEAN,
               handler: {o: Api::Node, m: :use_node_cache}
             )
             options.declare(:root_id, 'Gen4: File id of top folder when using access key (override AK root id)')
@@ -114,7 +139,7 @@ module Aspera
         SEARCH_REMOVE_FIELDS = %w[basename permissions].freeze
 
         # Actions in execute_command_gen3
-        COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download cat sync transport]
+        COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download cat sync transport spec]
 
         BASE_ACTIONS = %i[api_details].concat(COMMANDS_GEN3).freeze
 
@@ -142,9 +167,9 @@ module Aspera
         GEN4_LS_FIELDS = %w[name type recursive_size size modified_time access_level].freeze
 
         # @param api [Rest] an existing API object for the Node API
-        # @param prefix_path [String,nil] for Faspex 4, allows browsing a package
+        # @param prefix_path [String,nil] for Faspex 4, allows browsing a package without full path in node (removes storage prefix)
         def initialize(context:, api: nil, prefix_path: nil)
-          @prefix_path = prefix_path
+          @prefixer = prefix_path ? NodePathPrefix.new(prefix_path) : nil
           super(context: context, basic_options: api.nil?)
           Node.declare_options(options)
           return if context.only_manual?
@@ -171,43 +196,12 @@ module Aspera
             end
         end
 
-        # reduce the path from a result on given named column
-        def c_result_remove_prefix_path(result, column)
-          return result if @prefix_path.nil?
-          case result[:type]
-          when :object_list
-            result[:data].each do |item|
-              item[column] = item[column][@prefix_path.length..-1] if item[column].start_with?(@prefix_path)
-            end
-          when :single_object
-            item = result[:data]
-            item[column] = item[column][@prefix_path.length..-1] if item[column].start_with?(@prefix_path)
-          end
-          return result
-        end
-
-        # translates paths results into CLI result, and removes prefix
-        def c_result_translate_rem_prefix(response, type, success_msg)
-          errors = []
-          final_result = {type: :object_list, data: [], fields: [type, 'result']}
-          response['paths'].each do |p|
-            result = success_msg
-            if p.key?('error')
-              Log.log.error{"#{p['error']['user_message']} : #{p['path']}"}
-              result = p['error']['user_message']
-              errors.push([p['path'], p['error']['user_message']])
-            end
-            final_result[:data].push({type => p['path'], 'result' => result})
-          end
-          # one error make all fail
-          raise errors.map{ |i| "#{i.first}: #{i.last}"}.join(', ') unless errors.empty?
-          return c_result_remove_prefix_path(final_result, type)
-        end
-
         # Gen3 API
         def browse_gen3
-          folders_to_process = [get_one_argument_with_prefix('path')]
-          query = options.get_option(:query, default: {})
+          folders_to_process = options.get_next_argument('path', validation: String)
+          folders_to_process = @prefixer.add_to_path(folders_to_process) unless @prefixer.nil?
+          folders_to_process = [folders_to_process]
+          query = options.get_option(:query) || {}
           # special parameter: max number of entries in result
           max_items = query.delete(MAX_ITEMS)
           # special parameter: recursive browsing
@@ -225,14 +219,13 @@ module Aspera
             query['path'] = path
             offset = 0
             total_count = nil
-            result = nil
             loop do
               # example: send_result={'items'=>[{'file'=>"filename1","permissions"=>[{'name'=>'read'},{'name'=>'write'}]}]}
               response = @api_node.create('files/browse', query)
               # 'file','symbolic_link'
               if !Node.gen3_entry_folder?(response['self']) || only_path
-                result = {type: :single_object, data: response['self']}
-                break
+                @prefixer&.remove_in_object_list!([response['self']])
+                return Main.result_single_object(response['self'])
               end
               items = response['items']
               total_count ||= response['total_count']
@@ -253,9 +246,10 @@ module Aspera
             end
             query.delete('skip')
           end
-          result ||= {type: :object_list, data: all_items}
+          @prefixer&.remove_in_object_list!(all_items)
+          return Main.result_object_list(all_items)
+        ensure
           formatter.long_operation_terminated
-          return c_result_remove_prefix_path(result, 'path')
         end
 
         # Create async transfer spec request from direction and folders
@@ -294,48 +288,57 @@ module Aspera
           case command
           when :delete
             # TODO: add query for recursive
-            paths_to_delete = get_all_arguments_with_prefix('file list')
+            paths_to_delete = options.get_next_argument('file list', multiple: true)
+            @prefixer&.add_to_paths!(paths_to_delete)
             resp = @api_node.create('files/delete', {paths: paths_to_delete.map{ |i| {'path' => i.start_with?('/') ? i : "/#{i}"}}})
-            return c_result_translate_rem_prefix(resp, 'file', 'deleted')
+            return cli_result_from_paths_response(resp, 'file deleted')
           when :search
-            search_root = get_one_argument_with_prefix('search root')
+            search_root = options.get_next_argument('search root', validation: String)
+            search_root = @prefixer.add_to_path(search_root) unless @prefixer.nil?
             parameters = {'path' => search_root}
             other_options = options.get_option(:query)
             parameters.merge!(other_options) unless other_options.nil?
             resp = @api_node.create('files/search', parameters)
-            result = {type: :object_list, data: resp['items']}
-            return Main.result_empty if result[:data].empty?
-            result[:fields] = result[:data].first.keys.reject{ |i| SEARCH_REMOVE_FIELDS.include?(i)}
+            return Main.result_empty if resp['items'].empty?
+            fields = resp['items'].first.keys.reject{ |i| SEARCH_REMOVE_FIELDS.include?(i)}
             formatter.display_item_count(resp['item_count'], resp['total_count'])
             formatter.display_status("params: #{resp['parameters'].keys.map{ |k| "#{k}:#{resp['parameters'][k]}"}.join(',')}")
-            return c_result_remove_prefix_path(result, 'path')
+            @prefixer&.remove_in_object_list!(resp['items'])
+            return Main.result_object_list(resp['items'], fields: fields)
           when :space
-            path_list = get_all_arguments_with_prefix('folder path or ext.val. list')
+            path_list = options.get_next_argument('folder path or ext.val. list', multiple: true)
+            @prefixer&.add_to_paths!(path_list)
             resp = @api_node.create('space', {'paths' => path_list.map{ |i| {path: i}}})
-            result = {type: :object_list, data: resp['paths']}
-            # return c_result_translate_rem_prefix(resp,'folder','created',@prefix_path)
-            return c_result_remove_prefix_path(result, 'path')
+            @prefixer&.remove_in_object_list!(resp['paths'])
+            return Main.result_object_list(resp['paths'])
           when :mkdir
-            path_list = get_all_arguments_with_prefix('folder path or ext.val. list')
+            path_list = options.get_next_argument('folder path or ext.val. list', multiple: true)
+            @prefixer&.add_to_paths!(path_list)
             resp = @api_node.create('files/create', {'paths' => path_list.map{ |i| {type: :directory, path: i}}})
-            return c_result_translate_rem_prefix(resp, 'folder', 'created')
+            return cli_result_from_paths_response(resp, 'folder created')
           when :mklink
-            target = get_one_argument_with_prefix('target')
-            one_path = get_one_argument_with_prefix('link path')
+            target = options.get_next_argument('target', validation: String)
+            target = @prefixer.add_to_path(target) unless @prefixer.nil?
+            one_path = options.get_next_argument('link path', validation: String)
+            one_path = @prefixer.add_to_path(one_path) unless @prefixer.nil?
             resp = @api_node.create('files/create', {'paths' => [{type: :symbolic_link, path: one_path, target: {path: target}}]})
-            return c_result_translate_rem_prefix(resp, 'folder', 'created')
+            return cli_result_from_paths_response(resp, 'link created')
           when :mkfile
-            one_path = get_one_argument_with_prefix('file path')
+            one_path = options.get_next_argument('file path', validation: String)
+            one_path = @prefixer.add_to_path(one_path) unless @prefixer.nil?
             contents64 = Base64.strict_encode64(options.get_next_argument('contents'))
             resp = @api_node.create('files/create', {'paths' => [{type: :file, path: one_path, contents: contents64}]})
-            return c_result_translate_rem_prefix(resp, 'folder', 'created')
+            return cli_result_from_paths_response(resp, 'file created')
           when :rename
             # TODO: multiple ?
-            path_base = get_one_argument_with_prefix('path_base')
-            path_src = get_one_argument_with_prefix('path_src')
-            path_dst = get_one_argument_with_prefix('path_dst')
+            path_base = options.get_next_argument('path_base', validation: String)
+            path_base = @prefixer.add_to_path(path_base) unless @prefixer.nil?
+            path_src = options.get_next_argument('path_src', validation: String)
+            path_src = @prefixer.add_to_path(path_src) unless @prefixer.nil?
+            path_dst = options.get_next_argument('path_dst', validation: String)
+            path_dst = @prefixer.add_to_path(path_dst) unless @prefixer.nil?
             resp = @api_node.create('files/rename', {'paths' => [{'path' => path_base, 'source' => path_src, 'destination' => path_dst}]})
-            return c_result_translate_rem_prefix(resp, 'entry', 'moved')
+            return cli_result_from_paths_response(resp, 'entry moved')
           when :browse
             return browse_gen3
           when :sync
@@ -377,15 +380,15 @@ module Aspera
             transfer_spec.delete('paths') if command.eql?(:upload)
             return Main.result_transfer(transfer.start(transfer_spec))
           when :cat
-            remote_path = get_one_argument_with_prefix('remote path')
+            remote_path = options.get_next_argument('remote path', validation: String)
+            remote_path = @prefixer.add_to_path(remote_path) unless @prefixer.nil?
             File.basename(remote_path)
-            result = @api_node.call(
-              operation: 'GET',
-              subpath: "files/#{URI.encode_www_form_component(remote_path)}/contents"
-            )
-            return Main.result_text(result[:http].body)
+            http = @api_node.read("files/#{URI.encode_www_form_component(remote_path)}/contents", ret: :resp)
+            return Main.result_text(http.body)
           when :transport
             return Main.result_single_object(@api_node.transport_params)
+          when :spec
+            return Main.result_single_object(@api_node.base_spec)
           end
           Aspera.error_unreachable_line
         end
@@ -438,7 +441,7 @@ module Aspera
               nagios.add_ok('node api', 'accessible')
               nagios.check_time_offset(info['current_time'], 'node api')
               nagios.check_product_version('node api', 'entsrv', info['version'])
-            rescue RestCallError => e
+            rescue StandardError => e
               nagios.add_critical('node api', e.to_s)
             end
             begin
@@ -447,13 +450,14 @@ module Aspera
                 subpath:      'services/soap/Transfer-201210',
                 content_type: Rest::MIME_TEXT,
                 body:         CENTRAL_SOAP_API_TEST,
-                headers:      {'Content-Type' => 'text/xml;charset=UTF-8', 'SOAPAction' => 'FASPSessionNET-200911#GetSessionInfo'}
-              )[:http].body
+                headers:      {'Content-Type' => 'text/xml;charset=UTF-8', 'SOAPAction' => 'FASPSessionNET-200911#GetSessionInfo'},
+                ret:          :resp
+              ).body
               nagios.add_ok('central', 'accessible by node')
             rescue StandardError => e
               nagios.add_critical('central', e.to_s)
             end
-            return nagios.result
+            Main.result_object_list(nagios.status_list)
           when :events
             events = @api_node.read('events', query_read_delete)
             return Main.result_object_list(events, fields: ->(f){!f.start_with?('data')})
@@ -513,8 +517,8 @@ module Aspera
             else Aspera.error_unreachable_line
             end
             return Main.result_single_object(result) if command_repo.eql?(:node_info)
-            # check format of bearer token
-            OAuth::Factory.bearer_token(result[:password])
+            raise BadArgument, 'Cannot get bearer token if authenticating with secret' unless apifid[:api].auth_params[:type].eql?(:oauth2)
+            Aspera.assert(OAuth::Factory.bearer_auth?(result[:password])){'Not using bearer token auth'}
             return Main.result_text(result[:password])
           when :browse
             apifid = apifid_from_next_arg(top_file_id)
@@ -531,7 +535,7 @@ module Aspera
           when :mkdir, :mklink, :mkfile
             containing_folder_path, new_item = Api::Node.split_folder(options.get_next_argument('path'))
             apifid = @api_node.resolve_api_fid(top_file_id, containing_folder_path, true)
-            query = options.get_option(:query, mandatory: false)
+            query = options.get_option(:query)
             check_exists = true
             payload = {name: new_item}
             if query
@@ -570,10 +574,11 @@ module Aspera
             return Main.result_status("renamed to #{newname}")
           when :delete
             return do_bulk_operation(command: command_repo, descr: 'path', values: String, id_result: 'path') do |l_path|
-              apifid = if (m = l_path.match(REGEX_LOOKUP_ID_BY_FIELD))
+              apifid = if (m = Base.percent_selector(l_path))
+                Aspera.assert_values(m[:field], ['id'], type: BadIdentifier)
                 {
                   api:     @api_node,
-                  file_id: m[2]
+                  file_id: m[:value]
                 }
               else
                 @api_node.resolve_api_fid(top_file_id, l_path)
@@ -602,11 +607,8 @@ module Aspera
             return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Transfer::Spec::DIRECTION_RECEIVE, {'paths'=>source_paths})))
           when :cat
             apifid = apifid_from_next_arg(top_file_id)
-            result = apifid[:api].call(
-              operation: 'GET',
-              subpath: "files/#{apifid[:file_id]}/content"
-            )
-            return Main.result_text(result[:http].body)
+            http = apifid[:api].read("files/#{apifid[:file_id]}/content", ret: :resp)
+            return Main.result_text(http.body)
           when :show
             apifid = apifid_from_next_arg(top_file_id)
             items = apifid[:api].read("files/#{apifid[:file_id]}")
@@ -618,12 +620,8 @@ module Aspera
             return Main.result_status('Done')
           when :thumbnail
             apifid = apifid_from_next_arg(top_file_id)
-            result = apifid[:api].call(
-              operation: 'GET',
-              subpath: "files/#{apifid[:file_id]}/preview",
-              headers: {'Accept' => 'image/png'}
-            )
-            return Main.result_image(result[:http].body)
+            http = apifid[:api].read("files/#{apifid[:file_id]}/preview", headers: {'Accept' => 'image/png'}, ret: :resp)
+            return Main.result_image(http.body)
           when :permission
             apifid = apifid_from_next_arg(top_file_id)
             command_perm = options.get_next_command(%i[list show create delete])
@@ -677,7 +675,7 @@ module Aspera
           async_ids = @api_node.read('async/list')['sync_ids']
           summaries = @api_node.create('async/summary', {'syncs' => async_ids})['sync_summaries']
           selected = summaries.find{ |s| s['name'].eql?(value)}
-          raise Cli::BadIdentifier.new('sync', "#{field}=#{value}") if selected.nil?
+          raise Cli::BadIdentifier.new('sync', value, field: field) if selected.nil?
           return selected['snid']
         end
 
@@ -762,7 +760,7 @@ module Aspera
             # name is unique, so we can return
             return id if sync_info[field].eql?(value)
           end
-          raise Cli::BadIdentifier.new('ssync', "#{field}=#{value}")
+          raise Cli::BadIdentifier.new('ssync', value, field: field)
         end
 
         WATCH_FOLDER_MUL = %i[create list].freeze
@@ -789,7 +787,7 @@ module Aspera
           when :show
             return Main.result_single_object(@api_node.read(one_res_path))
           when :modify
-            @api_node.update(one_res_path, options.get_option(:query, mandatory: true))
+            @api_node.update(one_res_path, value_create_modify(command: 'watch_folder'))
             return Main.result_status("#{one_res_id} updated")
           when :delete
             @api_node.delete(one_res_path)
@@ -837,12 +835,12 @@ module Aspera
                   operation:    'POST',
                   subpath:      "asyncs/#{asyncs_id}/#{sync_command}",
                   content_type: Rest::MIME_TEXT,
-                  body:         ''
-                )[:http].body
+                  body:         '',
+                  ret:          :resp
+                ).body
                 return Main.result_status('Done')
               end
-              parameters = nil
-              parameters = options.get_option(:query, default: {}) if %i[bandwidth counters files].include?(sync_command)
+              parameters = options.get_option(:query) || {} if %i[bandwidth counters files].include?(sync_command)
               return Main.result_single_object(@api_node.read("asyncs/#{asyncs_id}/#{sync_command}", parameters))
             end
           when :stream
@@ -895,8 +893,9 @@ module Aspera
             when :sessions
               transfers_data = @api_node.read('ops/transfers', query_read_delete)
               sessions = transfers_data.flat_map{ |t| t['sessions']}
+              start_end = %i[start end].freeze
               sessions.each do |session|
-                %i[start end].each do |what|
+                start_end.each do |what|
                   session["#{what}_time"] = session["#{what}_time_usec"] ? Time.at(session["#{what}_time_usec"] / 1_000_000.0).utc.iso8601(0) : nil
                 end
               end
@@ -979,7 +978,7 @@ module Aspera
             command = options.get_next_command(%i[session file])
             validator_id = options.get_option(:validator)
             validation = {'validator_id' => validator_id} unless validator_id.nil?
-            request_data = options.get_option(:query, default: {})
+            request_data = options.get_option(:query) || {}
             case command
             when :session
               command = options.get_next_command([:list])
@@ -1112,18 +1111,31 @@ module Aspera
 
         private
 
-        # get remaining path arguments from command line, and add prefix
-        def get_all_arguments_with_prefix(name)
-          path_args = options.get_next_argument(name, multiple: true)
-          return path_args if @prefix_path.nil?
-          return path_args.map{ |p| File.join(@prefix_path, p)}
+        # Response has key `paths`.
+        # From those, check if there is an error
+        # @return [Array] of Hash with 2 keys: `path` and `result`
+        def response_to_result(response, success_msg)
+          errors = []
+          obj_list = []
+          response['paths'].each do |p|
+            result = success_msg
+            if p.key?('error')
+              Log.log.error{"#{p['error']['user_message']} : #{p['path']}"}
+              result = p['error']['user_message']
+              errors.push([p['path'], p['error']['user_message']])
+            end
+            obj_list.push({'path' => p['path'], 'result' => result})
+          end
+          # one error make all fail
+          raise errors.map{ |i| "#{i.first}: #{i.last}"}.join(', ') unless errors.empty?
+          obj_list
         end
 
-        # get next path argument from command line, and add prefix
-        def get_one_argument_with_prefix(name)
-          path_arg = options.get_next_argument(name, validation: String)
-          return path_arg if @prefix_path.nil?
-          return File.join(@prefix_path, path_arg)
+        # Translates paths results into CLI result, and removes prefix
+        def cli_result_from_paths_response(response, success_msg)
+          obj_list = response_to_result(response, success_msg)
+          @prefixer&.remove_in_object_list!(obj_list)
+          return Main.result_object_list(obj_list, fields: %w[path result])
         end
 
         # Executes the provided API call in loop
@@ -1140,13 +1152,13 @@ module Aspera
           item_list = []
           query_token[:iteration_token] = iteration[0] unless iteration.nil?
           loop do
-            result = api.call(**call_args, query: query_token)
-            Aspera.assert_type(result[:data], Array){"Expected data to be an Array, got: #{result[:data].class}"}
+            data, http = api.call(**call_args, query: query_token, ret: :both)
+            Aspera.assert_type(data, Array){"Expected data to be an Array, got: #{data.class}"}
             # no data
-            break if result[:data].empty?
+            break if data.empty?
             # get next iteration token from link
             next_iteration_token = nil
-            link_info = result[:http]['Link']
+            link_info = http['Link']
             unless link_info.nil?
               m = link_info.match(/<([^>]+)>/)
               Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
@@ -1155,7 +1167,7 @@ module Aspera
             # same as last iteration: stop
             break if next_iteration_token&.eql?(query_token[:iteration_token])
             query_token[:iteration_token] = next_iteration_token
-            item_list.concat(result[:data])
+            item_list.concat(data)
             if max&.<=(item_list.length)
               item_list = item_list.slice(0, max)
               break

data/lib/aspera/cli/plugins/oauth.rb

@@ -13,7 +13,7 @@ module Aspera
         AUTH_OPTIONS = %i[url auth client_id client_secret scope redirect_uri private_key passphrase username password].freeze
         def initialize(**_)
           super
-          options.declare(:auth, 'OAuth type of authentication', values: AUTH_TYPES, default: :jwt)
+          options.declare(:auth, 'OAuth type of authentication', allowed: AUTH_TYPES, default: :jwt)
           options.declare(:client_id, 'OAuth client identifier')
           options.declare(:client_secret, 'OAuth client secret')
           options.declare(:redirect_uri, 'OAuth (Web) redirect URI for web authentication')