aspera-cli 4.19.0 → 4.20.0

data/lib/aspera/cli/plugins/node.rb

@@ -22,6 +22,10 @@ module Aspera
       class Node < Cli::BasicAuthPlugin
         include SyncActions
         class << self
+          # directory: node, container: shares
+          FOLDER_TYPES = %w[directory container].freeze
+          private_constant :FOLDER_TYPES
+
           def application_name
             'HSTS Node API'
           end
@@ -43,9 +47,9 @@ module Aspera
               test_endpoint = 'ping'
               result = api.call(operation: 'GET', subpath: test_endpoint)
               next unless result[:http].body.eql?('')
-              url_length = -2 - test_endpoint.length
+              url_end = -2 - test_endpoint.length
               return {
-                url:     result[:http].uri.to_s[0..url_length],
+                url:     result[:http].uri.to_s[0..url_end],
                 version: 'requires authentication'
               }
             rescue StandardError => e
@@ -69,16 +73,25 @@ module Aspera
           end
 
           def declare_options(options)
+            return if @options_declared
+            @options_declared = true
             options.declare(:validator, 'Identifier of validator (optional for central)')
             options.declare(:asperabrowserurl, 'URL for simple aspera web ui', default: 'https://asperabrowser.mybluemix.net')
             options.declare(:sync_name, 'Sync name')
             options.declare(
               :default_ports, 'Use standard FASP ports or get from node api (gen4)', values: :bool, default: :yes,
               handler: {o: Api::Node, m: :use_standard_ports})
-            options.declare(:root_id, 'File id of top folder if using bearer tokens')
+            options.declare(
+              :node_cache, 'Set to no to force actual file system read (gen4)', values: :bool,
+              handler: {o: Api::Node, m: :use_node_cache})
+            options.declare(:root_id, 'File id of top folder when using access key (override AK root id)')
             SyncActions.declare_options(options)
             options.parse_options!
           end
+
+          def gen3_entry_folder?(entry)
+            FOLDER_TYPES.include?(entry['type'])
+          end
         end
 
         # spellchecker: disable
@@ -164,7 +177,7 @@ module Aspera
         def c_result_translate_rem_prefix(response, type, success_msg, path_prefix)
           errors = []
           final_result = {type: :object_list, data: [], fields: [type, 'result']}
-          JSON.parse(response[:http].body)['paths'].each do |p|
+          response['paths'].each do |p|
             result = success_msg
             if p.key?('error')
               Log.log.error{"#{p['error']['user_message']} : #{p['path']}"}
@@ -180,9 +193,6 @@ module Aspera
           return c_result_remove_prefix_path(final_result, type, path_prefix)
         end
 
-        # directory: node, container: shares
-        FOLDER_TYPE = %w[directory container].freeze
-
         def browse_gen3(prefix_path)
           folders_to_process = [get_one_argument_with_prefix(prefix_path, 'path')]
           query = options.get_option(:query, default: {})
@@ -206,26 +216,21 @@ module Aspera
             result = nil
             loop do
               # example: send_result={'items'=>[{'file'=>"filename1","permissions"=>[{'name'=>'read'},{'name'=>'write'}]}]}
-              response = @api_node.call(
-                operation:   'POST',
-                subpath:     'files/browse',
-                headers:     {'Accept' => 'application/json'},
-                body:        query,
-                body_type:   :json)
+              response = @api_node.create('files/browse', query)
               # 'file','symbolic_link'
-              if only_path || !FOLDER_TYPE.include?(response[:data]['self']['type'])
-                result = { type: :single_object, data: response[:data]['self']}
+              if !Node.gen3_entry_folder?(response['self']) || only_path
+                result = { type: :single_object, data: response['self']}
                 break
               end
-              items = response[:data]['items']
-              total_count ||= response[:data]['total_count']
+              items = response['items']
+              total_count ||= response['total_count']
               all_items.concat(items)
               if single_call
-                formatter.display_item_count(response[:data]['item_count'], total_count)
+                formatter.display_item_count(response['item_count'], total_count)
                 break
               end
               if recursive
-                folders_to_process.concat(items.select{|i|FOLDER_TYPE.include?(i['type'])}.map{|i|i['path']})
+                folders_to_process.concat(items.select{|i|Node.gen3_entry_folder?(i)}.map{|i|i['path']})
               end
               if !max_items.nil? && (all_items.count >= max_items)
                 all_items = all_items.slice(0, max_items) if all_items.count > max_items
@@ -239,6 +244,7 @@ module Aspera
             query.delete('skip')
           end
           result ||= {type: :object_list, data: all_items}
+          formatter.long_operation_terminated
           return c_result_remove_prefix_path(result, 'path', prefix_path)
         end
 
@@ -253,19 +259,19 @@ module Aspera
           when :search
             search_root = get_one_argument_with_prefix(prefix_path, 'search root')
             parameters = {'path' => search_root}
-            other_options = query_option
+            other_options = options.get_option(:query)
             parameters.merge!(other_options) unless other_options.nil?
             resp = @api_node.create('files/search', parameters)
-            result = { type: :object_list, data: resp[:data]['items']}
+            result = { type: :object_list, data: resp['items']}
             return Main.result_empty if result[:data].empty?
             result[:fields] = result[:data].first.keys.reject{|i|SEARCH_REMOVE_FIELDS.include?(i)}
-            formatter.display_item_count(resp[:data]['item_count'], resp[:data]['total_count'])
-            formatter.display_status("params: #{resp[:data]['parameters'].keys.map{|k|"#{k}:#{resp[:data]['parameters'][k]}"}.join(',')}")
+            formatter.display_item_count(resp['item_count'], resp['total_count'])
+            formatter.display_status("params: #{resp['parameters'].keys.map{|k|"#{k}:#{resp['parameters'][k]}"}.join(',')}")
             return c_result_remove_prefix_path(result, 'path', prefix_path)
           when :space
             path_list = get_all_arguments_with_prefix(prefix_path, 'folder path or ext.val. list')
             resp = @api_node.create('space', { 'paths' => path_list.map {|i| { path: i} } })
-            result = { type: :object_list, data: resp[:data]['paths']}
+            result = { type: :object_list, data: resp['paths']}
             # return c_result_translate_rem_prefix(resp,'folder','created',prefix_path)
             return c_result_remove_prefix_path(result, 'path', prefix_path)
           when :mkdir
@@ -311,7 +317,7 @@ module Aspera
               # prepare payload for single request
               setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
               # only one request, so only one answer
-              transfer_spec = @api_node.create('files/sync_setup', setup_payload)[:data]['transfer_specs'].first['transfer_spec']
+              transfer_spec = @api_node.create('files/sync_setup', setup_payload)['transfer_specs'].first['transfer_spec']
               # API returns null tag... but async does not like it
               transfer_spec.delete_if{ |_k, v| v.nil? }
               # delete this part, as the returned value contains only destination, and not sources
@@ -333,7 +339,7 @@ module Aspera
             # prepare payload for single request
             setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
             # only one request, so only one answer
-            transfer_spec = @api_node.create("files/#{command}_setup", setup_payload)[:data]['transfer_specs'].first['transfer_spec']
+            transfer_spec = @api_node.create("files/#{command}_setup", setup_payload)['transfer_specs'].first['transfer_spec']
             # delete this part, as the returned value contains only destination, and not sources
             transfer_spec.delete('paths') if command.eql?(:upload)
             return Main.result_transfer(transfer.start(transfer_spec))
@@ -363,13 +369,13 @@ module Aspera
             when *Plugin::ALL_OPS
               return entity_command(ak_command, @api_node, 'access_keys') do |field, value|
                        raise 'only selector: %id:self' unless field.eql?('id') && value.eql?('self')
-                       @api_node.read('access_keys/self')[:data]['id']
+                       @api_node.read('access_keys/self')['id']
                      end
             when :do
               access_key_id = options.get_next_argument('access key id')
               root_file_id = options.get_option(:root_id)
               if root_file_id.nil?
-                ak_info = @api_node.read("access_keys/#{access_key_id}")[:data]
+                ak_info = @api_node.read("access_keys/#{access_key_id}")
                 # change API credentials if different access key
                 if !access_key_id.eql?('self')
                   @api_node.auth_params[:username] = ak_info['id']
@@ -381,7 +387,7 @@ module Aspera
               return execute_command_gen4(command_repo, root_file_id)
             when :set_bearer_key
               access_key_id = options.get_next_argument('access key id')
-              access_key_id = @api_node.read('access_keys/self')[:data]['id'] if access_key_id.eql?('self')
+              access_key_id = @api_node.read('access_keys/self')['id'] if access_key_id.eql?('self')
               bearer_key_pem = options.get_next_argument('public or private RSA key PEM value', validation: String)
               key = OpenSSL::PKey.read(bearer_key_pem)
               key = key.public_key if key.private?
@@ -392,7 +398,7 @@ module Aspera
           when :health
             nagios = Nagios.new
             begin
-              info = @api_node.read('info')[:data]
+              info = @api_node.read('info')
               nagios.add_ok('node api', 'accessible')
               nagios.check_time_offset(info['current_time'], 'node api')
               nagios.check_product_version('node api', 'entsrv', info['version'])
@@ -412,17 +418,17 @@ module Aspera
             end
             return nagios.result
           when :events
-            events = @api_node.read('events', query_read_delete)[:data]
+            events = @api_node.read('events', query_read_delete)
             return { type: :object_list, data: events, fields: ->(f){!f.start_with?('data')} }
           when :info
-            nd_info = @api_node.read('info')[:data]
+            nd_info = @api_node.read('info')
             return { type: :single_object, data: nd_info}
           when :slash
-            nd_info = @api_node.read('')[:data]
+            nd_info = @api_node.read('')
             return { type: :single_object, data: nd_info}
           when :license
             # requires: asnodeadmin -mu <node user> --acl-add=internal --internal
-            node_license = @api_node.read('license')[:data]
+            node_license = @api_node.read('license')
             if node_license['failure'].is_a?(String) && node_license['failure'].include?('ACL')
               Log.log.error('server must have: asnodeadmin -mu <node user> --acl-add=internal --internal')
             end
@@ -432,8 +438,8 @@ module Aspera
           end
         end
 
-        # @return [Hash] api and main file id for given path or id
-        # Allows to specify a file by its path or by its id on the node
+        # Allows to specify a file by its path or by its id on the node in command line
+        # @return [Hash] api and main file id for given path or id in next argument
         def apifid_from_next_arg(top_file_id)
           file_path = instance_identifier(description: 'path or %id:<id>') do |attribute, value|
             raise 'Only selection "id" is supported (file id)' unless attribute.eql?('id')
@@ -445,6 +451,9 @@ module Aspera
         end
 
         def execute_command_gen4(command_repo, top_file_id)
+          override_file_id = options.get_option(:root_id)
+          top_file_id = override_file_id unless override_file_id.nil?
+          raise 'Specify root file id with option root_id' if top_file_id.nil?
           case command_repo
           when :v3
             # NOTE: other common actions are unauthorized with user scope
@@ -453,7 +462,7 @@ module Aspera
             apifid = @api_node.resolve_api_fid(top_file_id, '')
             return Node.new(**init_params, api: apifid[:api]).execute_action(command_legacy)
           when :node_info, :bearer_token_node
-            apifid = @api_node.resolve_api_fid(top_file_id, options.get_next_argument('path'))
+            apifid = apifid_from_next_arg(top_file_id)
             result = {
               url:     apifid[:api].base_url,
               root_id: apifid[:file_id]
@@ -473,10 +482,14 @@ module Aspera
             OAuth::Factory.bearer_extract(result[:password])
             return Main.result_status(result[:password])
           when :browse
-            apifid = @api_node.resolve_api_fid(top_file_id, options.get_next_argument('path'))
-            file_info = apifid[:api].read("files/#{apifid[:file_id]}")[:data]
+            apifid = apifid_from_next_arg(top_file_id)
+            file_info = apifid[:api].read_with_cache("files/#{apifid[:file_id]}")
             if file_info['type'].eql?('folder')
-              result = apifid[:api].read("files/#{apifid[:file_id]}/files", query_read_delete)
+              result = apifid[:api].call(
+                operation: 'GET',
+                subpath:   "files/#{apifid[:file_id]}/files",
+                headers:   Api::Node.cache_control_headers,
+                query:     query_read_delete)
               items = result[:data]
               formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count'])
             else
@@ -484,25 +497,33 @@ module Aspera
             end
             return {type: :object_list, data: items, fields: %w[name type recursive_size size modified_time access_level]}
           when :find
-            apifid = @api_node.resolve_api_fid(top_file_id, options.get_next_argument('path'))
+            apifid = apifid_from_next_arg(top_file_id)
             test_block = Api::Node.file_matcher_from_argument(options)
             return {type: :object_list, data: @api_node.find_files(apifid[:file_id], test_block), fields: ['path']}
           when :mkdir
             containing_folder_path = options.get_next_argument('path').split(Api::Node::PATH_SEPARATOR)
             new_folder = containing_folder_path.pop
-            apifid = @api_node.resolve_api_fid(top_file_id, containing_folder_path.join(Api::Node::PATH_SEPARATOR))
-            result = apifid[:api].create("files/#{apifid[:file_id]}/files", {name: new_folder, type: :folder})[:data]
+            # add trailing slash to force last link to be resolved
+            apifid = @api_node.resolve_api_fid(top_file_id, containing_folder_path.join(Api::Node::PATH_SEPARATOR), true)
+            result = apifid[:api].create("files/#{apifid[:file_id]}/files", {name: new_folder, type: :folder})
             return Main.result_status("created: #{result['name']} (id=#{result['id']})")
           when :rename
             file_path = options.get_next_argument('source path')
             apifid = @api_node.resolve_api_fid(top_file_id, file_path)
             newname = options.get_next_argument('new name')
-            result = apifid[:api].update("files/#{apifid[:file_id]}", {name: newname})[:data]
+            result = apifid[:api].update("files/#{apifid[:file_id]}", {name: newname})
             return Main.result_status("renamed to #{newname}")
           when :delete
             return do_bulk_operation(command: command_repo, descr: 'path', values: String, id_result: 'path') do |l_path|
-              apifid = @api_node.resolve_api_fid(top_file_id, l_path)
-              result = apifid[:api].delete("files/#{apifid[:file_id]}")[:data]
+              apifid = if (m = l_path.match(REGEX_LOOKUP_ID_BY_FIELD))
+                {
+                  api:     @api_node,
+                  file_id: m[2]
+                }
+              else
+                @api_node.resolve_api_fid(top_file_id, l_path)
+              end
+              result = apifid[:api].delete("files/#{apifid[:file_id]}")
               {'path' => l_path}
             end
           when :sync
@@ -522,27 +543,24 @@ module Aspera
               transfer_spec
             end
           when :upload
-            apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Transfer::Spec::DIRECTION_SEND))
+            apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Transfer::Spec::DIRECTION_SEND), true)
             return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Transfer::Spec::DIRECTION_SEND)))
           when :download
             source_paths = transfer.ts_source_paths
             # special case for AoC : all files must be in same folder
             source_folder = source_paths.shift['source']
             # if a single file: split into folder and path
-            apifid = @api_node.resolve_api_fid(top_file_id, source_folder)
+            apifid = @api_node.resolve_api_fid(top_file_id, source_folder, true)
             if source_paths.empty?
               # get precise info in this element
-              file_info = apifid[:api].read("files/#{apifid[:file_id]}")[:data]
+              file_info = apifid[:api].read("files/#{apifid[:file_id]}")
               case file_info['type']
               when 'file'
                 # if the single source is a file, we need to split into folder path and filename
                 src_dir_elements = source_folder.split(Api::Node::PATH_SEPARATOR)
                 # filename is the last one, source folder is what remains
                 source_paths = [{'source' => src_dir_elements.pop}]
-                # add trailing / so that link is resolved, if it's a shared folder
-                src_dir_elements.push('')
-                source_folder = src_dir_elements.join(Api::Node::PATH_SEPARATOR)
-                apifid = @api_node.resolve_api_fid(top_file_id, source_folder)
+                apifid = @api_node.resolve_api_fid(top_file_id, src_dir_elements.join(Api::Node::PATH_SEPARATOR), true)
               when 'link', 'folder'
                 # single source is 'folder' or 'link'
                 # TODO: add this ? , 'destination'=>file_info['name']
@@ -570,12 +588,12 @@ module Aspera
             return Main.result_status("downloaded: #{file_name}")
           when :show
             apifid = apifid_from_next_arg(top_file_id)
-            items = apifid[:api].read("files/#{apifid[:file_id]}")[:data]
+            items = apifid[:api].read("files/#{apifid[:file_id]}")
             return {type: :single_object, data: items}
           when :modify
             apifid = apifid_from_next_arg(top_file_id)
             update_param = options.get_next_argument('update data', validation: Hash)
-            apifid[:api].update("files/#{apifid[:file_id]}", update_param)[:data]
+            apifid[:api].update("files/#{apifid[:file_id]}", update_param)
             return Main.result_status('Done')
           when :thumbnail
             apifid = apifid_from_next_arg(top_file_id)
@@ -595,13 +613,14 @@ module Aspera
               # add which one to get
               list_options['file_id'] = apifid[:file_id]
               list_options['inherited'] ||= false
-              items = apifid[:api].read('permissions', list_options)[:data]
+              items = apifid[:api].read('permissions', list_options)
               return {type: :object_list, data: items}
             when :delete
               return do_bulk_operation(command: command_perm, descr: 'identifier', values: :identifier) do |one_id|
                 apifid[:api].delete("permissions/#{one_id}")
                 # notify application of deletion
-                the_app[:api].permissions_send_event(created_data: created_data, app_info: the_app, types: ['permission.deleted']) unless the_app.nil?
+                the_app = apifid[:api].app_info
+                the_app&.[](:api)&.permissions_send_event(event_data: {}, app_info: the_app, types: ['permission.deleted'])
                 {'id' => one_id}
               end
             when :create
@@ -611,11 +630,11 @@ module Aspera
               create_param['access_levels'] = Api::Node::ACCESS_LEVELS unless create_param.key?('access_levels')
               # add application specific tags (AoC)
               the_app = apifid[:api].app_info
-              the_app[:api].permissions_set_create_params(create_param: create_param, app_info: the_app) unless the_app.nil?
+              the_app&.[](:api)&.permissions_set_create_params(perm_data: create_param, app_info: the_app)
               # create permission
-              created_data = apifid[:api].create('permissions', create_param)[:data]
+              created_data = apifid[:api].create('permissions', create_param)
               # notify application of creation
-              the_app[:api].permissions_send_event(created_data: created_data, app_info: the_app) unless the_app.nil?
+              the_app&.[](:api)&.permissions_send_event(event_data: created_data, app_info: the_app)
               return { type: :single_object, data: created_data}
             else Aspera.error_unreachable_line
             end
@@ -632,14 +651,14 @@ module Aspera
             if async_name.nil?
               async_id = instance_identifier
               if async_id.eql?(SpecialValues::ALL) && %i[show delete].include?(command)
-                async_ids = @api_node.read('async/list')[:data]['sync_ids']
+                async_ids = @api_node.read('async/list')['sync_ids']
               else
                 Integer(async_id) # must be integer
                 async_ids = [async_id]
               end
             else
-              async_ids = @api_node.read('async/list')[:data]['sync_ids']
-              summaries = @api_node.create('async/summary', {'syncs' => async_ids})[:data]['sync_summaries']
+              async_ids = @api_node.read('async/list')['sync_ids']
+              summaries = @api_node.create('async/summary', {'syncs' => async_ids})['sync_summaries']
               selected = summaries.find{|s|s['name'].eql?(async_name)}
               raise "no such sync: #{async_name}" if selected.nil?
               async_id = selected['snid']
@@ -649,19 +668,19 @@ module Aspera
           end
           case command
           when :list
-            resp = @api_node.read('async/list')[:data]['sync_ids']
+            resp = @api_node.read('async/list')['sync_ids']
             return { type: :value_list, data: resp, name: 'id' }
           when :show
-            resp = @api_node.create('async/summary', post_data)[:data]['sync_summaries']
+            resp = @api_node.create('async/summary', post_data)['sync_summaries']
             return Main.result_empty if resp.empty?
             return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if async_id.eql?(SpecialValues::ALL)
             return { type: :single_object, data: resp.first }
           when :delete
-            resp = @api_node.create('async/delete', post_data)[:data]
+            resp = @api_node.create('async/delete', post_data)
             return { type: :single_object, data: resp, name: 'id' }
           when :bandwidth
             post_data['seconds'] = 100 # TODO: as parameter with --value
-            resp = @api_node.create('async/bandwidth', post_data)[:data]
+            resp = @api_node.create('async/bandwidth', post_data)
             data = resp['bandwidth_data']
             return Main.result_empty if data.empty?
             data = data.first[async_id]['data']
@@ -671,9 +690,9 @@ module Aspera
             # filename str
             # skip int
             # status int
-            filter = query_option
+            filter = options.get_option(:query)
             post_data.merge!(filter) unless filter.nil?
-            resp = @api_node.create('async/files', post_data)[:data]
+            resp = @api_node.create('async/files', post_data)
             data = resp['sync_files']
             data = data.first[async_id] unless data.empty?
             iteration_data = []
@@ -696,7 +715,7 @@ module Aspera
             skip_ids_persistency&.save
             return { type: :object_list, data: data, name: 'id' }
           when :counters
-            resp = @api_node.create('async/counters', post_data)[:data]['sync_counters'].first[async_id].last
+            resp = @api_node.create('async/counters', post_data)['sync_counters'].first[async_id].last
             return Main.result_empty if resp.nil?
             return { type: :single_object, data: resp }
           end
@@ -708,8 +727,8 @@ module Aspera
         # @param [String] value value of the field to search
         def ssync_lookup(field, value)
           raise Cli::BadArgument, "Only search by name is supported (#{field})" unless field.eql?('name')
-          @api_node.read('asyncs')[:data]['ids'].each do |id|
-            sync_info = @api_node.read("asyncs/#{id}")[:data]['configuration']
+          @api_node.read('asyncs')['ids'].each do |id|
+            sync_info = @api_node.read("asyncs/#{id}")['configuration']
             # name is unique, so we can return
             return id if sync_info[field].eql?(value)
           end
@@ -750,27 +769,27 @@ module Aspera
                 return Main.result_status('Done')
               end
               parameters = nil
-              parameters = query_option(default: {}) if %i[bandwidth counters files].include?(sync_command)
-              return { type: :single_object, data: @api_node.read("asyncs/#{asyncs_id}/#{sync_command}", parameters)[:data] }
+              parameters = options.get_option(:query, default: {}) if %i[bandwidth counters files].include?(sync_command)
+              return { type: :single_object, data: @api_node.read("asyncs/#{asyncs_id}/#{sync_command}", parameters) }
             end
           when :stream
             command = options.get_next_command(%i[list create show modify cancel])
             case command
             when :list
               resp = @api_node.read('ops/transfers', query_read_delete)
-              return { type: :object_list, data: resp[:data], fields: %w[id status] } # TODO: useful?
+              return { type: :object_list, data: resp, fields: %w[id status] } # TODO: useful?
             when :create
               resp = @api_node.create('streams', value_create_modify(command: command))
-              return { type: :single_object, data: resp[:data] }
+              return { type: :single_object, data: resp }
             when :show
               resp = @api_node.read("ops/transfers/#{options.get_next_argument('transfer id')}")
-              return { type: :other_struct, data: resp[:data] }
+              return { type: :other_struct, data: resp }
             when :modify
               resp = @api_node.update("streams/#{options.get_next_argument('transfer id')}", value_create_modify(command: command))
-              return { type: :other_struct, data: resp[:data] }
+              return { type: :other_struct, data: resp }
             when :cancel
               resp = @api_node.cancel("streams/#{options.get_next_argument('transfer id')}")
-              return { type: :other_struct, data: resp[:data] }
+              return { type: :other_struct, data: resp }
             else
               raise 'error'
             end
@@ -783,14 +802,50 @@ module Aspera
             end
             case command
             when :list
-              transfers_data = @api_node.read(res_class_path, query_read_delete)[:data]
+              iteration_persistency = nil
+              iteration_data = []
+              if options.get_option(:once_only, mandatory: true)
+                iteration_persistency = PersistencyActionOnce.new(
+                  manager: persistency,
+                  data:    iteration_data,
+                  id:      IdGenerator.from_list([
+                    'node_transfers',
+                    options.get_option(:url, mandatory: true),
+                    options.get_option(:username, mandatory: true)
+                  ]))
+              end
+              transfer_filter = query_read_delete(default: {})
+              if transfer_filter.delete('reset')
+                iteration_data.clear
+                iteration_persistency&.save
+                return Main.result_status('Persistency reset')
+              end
+              max_items = transfer_filter.delete(MAX_ITEMS)
+              transfer_filter['iteration_token'] = iteration_persistency.data[0] unless iteration_data.empty?
+              transfers_data = []
+              loop do
+                result = @api_node.call(operation: 'GET', subpath: res_class_path, query: transfer_filter)
+                data = result[:data]
+                transfers_data.concat(data)
+                if !max_items.nil? && (transfers_data.length >= max_items)
+                  transfers_data = transfers_data.slice(0, max_items)
+                  break
+                end
+                link_info = result[:http]['Link']
+                break if iteration_persistency.nil? || data.empty? || link_info.nil?
+                m = link_info.match(/<([^>]+)>/)
+                raise "Problem with iteration: #{link_info}" if m.nil?
+                iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
+                iteration_data[0] = transfer_filter['iteration_token'] = iteration_token
+              end
+              iteration_persistency&.save
               return {
                 type:   :object_list,
                 data:   transfers_data,
                 fields: %w[id status start_spec.direction start_spec.remote_user start_spec.remote_host start_spec.destination_path]
               }
             when :sessions
-              transfers_data = @api_node.read(res_class_path, query_read_delete)[:data]
+              transfers_data = @api_node.read(res_class_path, query_read_delete)
               sessions = transfers_data.map{|t|t['sessions']}.flatten
               sessions.each do |session|
                 session['start_time'] = Time.at(session['start_time_usec'] / 1_000_000.0).utc.iso8601(0)
@@ -803,15 +858,15 @@ module Aspera
               }
             when :cancel
               resp = @api_node.cancel(one_res_path)
-              return { type: :other_struct, data: resp[:data] }
+              return { type: :other_struct, data: resp }
             when :show
               resp = @api_node.read(one_res_path)
-              return { type: :other_struct, data: resp[:data] }
+              return { type: :other_struct, data: resp }
             when :modify
               resp = @api_node.update(one_res_path, options.get_next_argument('update value', validation: Hash))
-              return { type: :other_struct, data: resp[:data] }
+              return { type: :other_struct, data: resp }
             when :bandwidth_average
-              transfers_data = @api_node.read(res_class_path, query_read_delete)[:data]
+              transfers_data = @api_node.read(res_class_path, query_read_delete)
               # collect all key dates
               bandwidth_period = {}
               dir_info = %i[avg_kbps sessions].freeze
@@ -866,12 +921,12 @@ module Aspera
             case command
             when :list
               resp = @api_node.read('rund/services')
-              return { type: :object_list, data: resp[:data]['services'] }
+              return { type: :object_list, data: resp['services'] }
             when :create
               # @json:'{"type":"WATCHFOLDERD","run_as":{"user":"user1"}}'
               params = options.get_next_argument('creation data', validation: Hash)
               resp = @api_node.create('rund/services', params)
-              return Main.result_status("#{resp[:data]['id']} created")
+              return Main.result_status("#{resp['id']} created")
             when :delete
               @api_node.delete("rund/services/#{service_id}")
               return Main.result_status("#{service_id} deleted")
@@ -889,36 +944,37 @@ module Aspera
             case command
             when :create
               resp = @api_node.create(res_class_path, value_create_modify(command: command))
-              return Main.result_status("#{resp[:data]['id']} created")
+              return Main.result_status("#{resp['id']} created")
             when :list
               resp = @api_node.read(res_class_path, query_read_delete)
-              return { type: :value_list, data: resp[:data]['ids'], name: 'id' }
+              return { type: :value_list, data: resp['ids'], name: 'id' }
             when :show
-              return { type: :single_object, data: @api_node.read(one_res_path)[:data]}
+              return { type: :single_object, data: @api_node.read(one_res_path)}
             when :modify
-              @api_node.update(one_res_path, query_option(mandatory: true))
+              @api_node.update(one_res_path, options.get_option(:query, mandatory: true))
               return Main.result_status("#{one_res_id} updated")
             when :delete
               @api_node.delete(one_res_path)
               return Main.result_status("#{one_res_id} deleted")
             when :state
-              return { type: :single_object, data: @api_node.read("#{one_res_path}/state")[:data] }
+              return { type: :single_object, data: @api_node.read("#{one_res_path}/state") }
             end
           when :central
             command = options.get_next_command(%i[session file])
             validator_id = options.get_option(:validator)
             validation = {'validator_id' => validator_id} unless validator_id.nil?
-            request_data = query_option(default: {})
+            request_data = options.get_option(:query, default: {})
             case command
             when :session
               command = options.get_next_command([:list])
               case command
               when :list
+                request_data = options.get_next_argument('request data', mandatory: false, validation: Hash, default: {})
                 request_data.deep_merge!({'validation' => validation}) unless validation.nil?
                 resp = @api_node.create('services/rest/transfers/v1/sessions', request_data)
                 return {
                   type:   :object_list,
-                  data:   resp[:data]['session_info_result']['session_info'],
+                  data:   resp['session_info_result']['session_info'],
                   fields: %w[session_uuid status transport direction bytes_transferred]
                 }
               end
@@ -926,12 +982,14 @@ module Aspera
               command = options.get_next_command(%i[list modify])
               case command
               when :list
+                request_data = options.get_next_argument('request data', mandatory: false, validation: Hash, default: {})
                 request_data.deep_merge!({'validation' => validation}) unless validation.nil?
-                resp = @api_node.create('services/rest/transfers/v1/files', request_data)[:data]
+                resp = @api_node.create('services/rest/transfers/v1/files', request_data)
                 resp = JSON.parse(resp) if resp.is_a?(String)
                 Log.log.debug{Log.dump(:resp, resp)}
                 return { type: :object_list, data: resp['file_transfer_info_result']['file_transfer_info'], fields: %w[session_uuid file_id status path]}
               when :modify
+                request_data = options.get_next_argument('request data', mandatory: false, validation: Hash, default: {})
                 request_data.deep_merge!(validation) unless validation.nil?
                 @api_node.update('services/rest/transfers/v1/files', request_data)
                 return Main.result_status('updated')