aspera-cli 4.12.0 → 4.14.0

data/lib/aspera/cli/plugins/node.rb

@@ -7,6 +7,7 @@ require 'aspera/hash_ext'
 require 'aspera/id_generator'
 require 'aspera/node'
 require 'aspera/aoc'
+require 'aspera/sync'
 require 'aspera/fasp/transfer_spec'
 require 'base64'
 require 'zlib'
@@ -14,6 +15,51 @@ require 'zlib'
 module Aspera
   module Cli
     module Plugins
+      class SyncSpecGen3
+        def initialize(api_node)
+          @api_node = api_node
+        end
+
+        def transfer_spec(direction, local_path, remote_path)
+          # empty transfer spec for authorization request
+          direction_sym = direction.to_sym
+          request_transfer_spec = {
+            type:  Aspera::Sync::DIRECTION_TO_REQUEST_TYPE[direction_sym],
+            paths: {
+              source:      remote_path,
+              destination: local_path
+            }
+          }
+          # add fixed parameters if any (for COS)
+          @api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
+          # prepare payload for single request
+          setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
+          # only one request, so only one answer
+          transfer_spec = @api_node.create('files/sync_setup', setup_payload)[:data]['transfer_specs'].first['transfer_spec']
+          # API returns null tag... but async does not like it
+          transfer_spec.delete_if{ |_k, v| v.nil? }
+          # delete this part, as the returned value contains only destination, and not sources
+          # transfer_spec.delete('paths') if command.eql?(:upload)
+          Log.dump(:ts, transfer_spec)
+          return transfer_spec
+        end
+      end
+
+      class SyncSpecGen4
+        def initialize(api_node, top_file_id)
+          @api_node = api_node
+          @top_file_id = top_file_id
+        end
+
+        def transfer_spec(direction, local_path, remote_path)
+          # remote is specified by option to_folder
+          apifid = @api_node.resolve_api_fid(@top_file_id, remote_path)
+          transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)
+          Log.dump(:ts, transfer_spec)
+          return transfer_spec
+        end
+      end
+
       class Node < Aspera::Cli::BasicAuthPlugin
         class << self
           def detect(base_url)
@@ -26,15 +72,10 @@ module Aspera
           end
 
           def register_node_options(env)
-            env[:options].add_opt_simple(:validator, 'identifier of validator (optional for central)')
-            env[:options].add_opt_simple(:asperabrowserurl, 'URL for simple aspera web ui')
-            env[:options].add_opt_simple(:sync_name, 'sync name')
-            env[:options].add_opt_simple(:path, 'file or folder path for gen4 operation "file"')
-            env[:options].add_opt_list(:token_type, %i[aspera basic hybrid], 'Type of token used for transfers')
-            env[:options].add_opt_boolean(:default_ports, 'use standard FASP ports or get from node api (gen4)')
-            env[:options].set_option(:asperabrowserurl, 'https://asperabrowser.mybluemix.net')
-            env[:options].set_option(:token_type, :aspera)
-            env[:options].set_option(:default_ports, :yes)
+            env[:options].declare(:validator, 'Identifier of validator (optional for central)')
+            env[:options].declare(:asperabrowserurl, 'URL for simple aspera web ui', default: 'https://asperabrowser.mybluemix.net')
+            env[:options].declare(:sync_name, 'Sync name')
+            env[:options].declare(:default_ports, 'Use standard FASP ports or get from node api (gen4)', values: :bool, default: :yes)
             env[:options].parse_options!
             Aspera::Node.use_standard_ports = env[:options].get_option(:default_ports)
           end
@@ -53,7 +94,7 @@ module Aspera
         SEARCH_REMOVE_FIELDS = %w[basename permissions].freeze
 
         # actions in execute_command_gen3
-        COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download]
+        COMMANDS_GEN3 = %i[search space mkdir mklink mkfile rename delete browse upload download sync]
 
         BASE_ACTIONS = %i[api_details].concat(COMMANDS_GEN3).freeze
 
@@ -71,7 +112,7 @@ module Aspera
         NODE4_READ_ACTIONS = %i[bearer_token_node node_info browse find].freeze
 
         # commands for execute_command_gen4
-        COMMANDS_GEN4 = %i[mkdir rename delete upload download sync http_node_download file v3].concat(NODE4_READ_ACTIONS).freeze
+        COMMANDS_GEN4 = %i[mkdir rename delete upload download sync http_node_download show modify permission thumbnail v3].concat(NODE4_READ_ACTIONS).freeze
 
         COMMANDS_COS = %i[upload download info access_key api_details transfer].freeze
         COMMANDS_SHARES = (BASE_ACTIONS - %i[search]).freeze
@@ -85,23 +126,23 @@ module Aspera
             if env.key?(:node_api)
               # this can be Aspera::Node or Aspera::Rest (shares)
               env[:node_api]
-            elsif options.get_option(:password, is_type: :mandatory).start_with?('Bearer ')
+            elsif options.get_option(:password, mandatory: true).start_with?('Bearer ')
               # info is provided like node_info of aoc
               Aspera::Node.new(params: {
-                base_url: options.get_option(:url, is_type: :mandatory),
+                base_url: options.get_option(:url, mandatory: true),
                 headers:  {
-                  Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => options.get_option(:username, is_type: :mandatory),
-                  'Authorization'                          => options.get_option(:password, is_type: :mandatory)
+                  Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => options.get_option(:username, mandatory: true),
+                  'Authorization'                          => options.get_option(:password, mandatory: true)
                 }
               })
             else
               # this is normal case
               Aspera::Node.new(params: {
-                base_url: options.get_option(:url, is_type: :mandatory),
+                base_url: options.get_option(:url, mandatory: true),
                 auth:     {
                   type:     :basic,
-                  username: options.get_option(:username, is_type: :mandatory),
-                  password: options.get_option(:password, is_type: :mandatory)
+                  username: options.get_option(:username, mandatory: true),
+                  password: options.get_option(:password, mandatory: true)
                 }})
             end
         end
@@ -145,7 +186,7 @@ module Aspera
         # translates paths results into CLI result, and removes prefix
         def c_result_translate_rem_prefix(response, type, success_msg, path_prefix)
           errors = []
-          resres = { data: [], type: :object_list, fields: [type, 'result']}
+          final_result = { data: [], type: :object_list, fields: [type, 'result']}
           JSON.parse(response[:http].body)['paths'].each do |p|
             result = success_msg
             if p.key?('error')
@@ -153,13 +194,13 @@ module Aspera
               result = 'ERROR: ' + p['error']['user_message']
               errors.push([p['path'], p['error']['user_message']])
             end
-            resres[:data].push({type => p['path'], 'result' => result})
+            final_result[:data].push({type => p['path'], 'result' => result})
           end
           # one error make all fail
           unless errors.empty?
             raise errors.map{|i|"#{i.first}: #{i.last}"}.join(', ')
           end
-          return c_result_remove_prefix_path(resres, type, path_prefix)
+          return c_result_remove_prefix_path(final_result, type, path_prefix)
         end
 
         # get path arguments from command line, and add prefix
@@ -181,13 +222,13 @@ module Aspera
           when :search
             search_root = get_next_arg_add_prefix(prefix_path, 'search root')
             parameters = {'path' => search_root}
-            other_options = options.get_option(:value)
+            other_options = value_or_query(allowed_types: Hash)
             parameters.merge!(other_options) unless other_options.nil?
             resp = @api_node.create('files/search', parameters)
             result = { type: :object_list, data: resp[:data]['items']}
             return Main.result_empty if result[:data].empty?
             result[:fields] = result[:data].first.keys.reject{|i|SEARCH_REMOVE_FIELDS.include?(i)}
-            formatter.display_status("Items: #{resp[:data]['item_count']}/#{resp[:data]['total_count']}")
+            formatter.display_item_count(resp[:data]['item_count'], resp[:data]['total_count'])
             formatter.display_status("params: #{resp[:data]['parameters'].keys.map{|k|"#{k}:#{resp[:data]['parameters'][k]}"}.join(',')}")
             return c_result_remove_prefix_path(result, 'path', prefix_path)
           when :space
@@ -228,44 +269,33 @@ module Aspera
             case send_result['self']['type']
             when 'directory', 'container' # directory: node, container: shares
               result = { data: send_result['items'], type: :object_list, textify: lambda { |table_data| c_textify_browse(table_data) } }
-              formatter.display_status("Items: #{send_result['item_count']}/#{send_result['total_count']}")
+              formatter.display_item_count(send_result['item_count'], send_result['total_count'])
             else # 'file','symbolic_link'
               result = { data: send_result['self'], type: :single_object}
               # result={ data: [send_result['self']] , type: :object_list, textify: lambda { |table_data| c_textify_browse(table_data) } }
               # raise "unknown type: #{send_result['self']['type']}"
             end
             return c_result_remove_prefix_path(result, 'path', prefix_path)
+          when :sync
+            node_sync = SyncSpecGen3.new(@api_node)
+            return Plugins::Sync.new(@agents, sync_spec: node_sync).execute_action
           when :upload, :download
-            token_type = options.get_option(:token_type)
-            # nil if Shares 1.x
-            token_type = :aspera if token_type.nil?
-            case token_type
-            when :aspera, :hybrid
-              # empty transfer spec for authorization request
-              request_transfer_spec = {}
-              # set requested paths depending on direction
-              request_transfer_spec[:paths] = command.eql?(:download) ? transfer.ts_source_paths : [{ destination: transfer.destination_folder('send') }]
-              # add fixed parameters if any (for COS)
-              @api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
-              # prepare payload for single request
-              setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
-              # only one request, so only one answer
-              transfer_spec = @api_node.create("files/#{command}_setup", setup_payload)[:data]['transfer_specs'].first['transfer_spec']
-              # delete this part, as the returned value contains only destination, and not sources
-              transfer_spec.delete('paths') if command.eql?(:upload)
-            when :basic
-              raise 'shall have auth' unless @api_node.params[:auth].is_a?(Hash)
-              raise 'shall be basic auth' unless @api_node.params[:auth][:type].eql?(:basic)
-              transfer_spec = {}.merge(Aspera::Fasp::TransferSpec::AK_TSPEC_BASE)
-              transfer_spec['remote_host'] = URI.parse(@api_node.params[:base_url]).host
-              Fasp::TransferSpec.action_to_direction(transfer_spec, command)
-              transfer_spec['destination_root'] = transfer.destination_folder(transfer_spec['direction'])
-              @api_node.add_tspec_info(transfer_spec) if @api_node.respond_to?(:add_tspec_info)
-            else raise "ERROR: token_type #{tt}"
-            end
-            if %i[basic hybrid].include?(token_type)
-              @api_node.ts_basic_token(transfer_spec)
+            # empty transfer spec for authorization request
+            request_transfer_spec = {}
+            # set requested paths depending on direction
+            request_transfer_spec[:paths] = if command.eql?(:download)
+              transfer.ts_source_paths
+            else
+              [{ destination: transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND) }]
             end
+            # add fixed parameters if any (for COS)
+            @api_node.add_tspec_info(request_transfer_spec) if @api_node.respond_to?(:add_tspec_info)
+            # prepare payload for single request
+            setup_payload = {transfer_requests: [{transfer_request: request_transfer_spec}]}
+            # only one request, so only one answer
+            transfer_spec = @api_node.create("files/#{command}_setup", setup_payload)[:data]['transfer_specs'].first['transfer_spec']
+            # delete this part, as the returned value contains only destination, and not sources
+            transfer_spec.delete('paths') if command.eql?(:upload)
             return Main.result_transfer(transfer.start(transfer_spec))
           end
           raise 'INTERNAL ERROR'
@@ -314,7 +344,7 @@ module Aspera
             end
             return nagios.result
           when :events
-            events = @api_node.read('events', options.get_option(:value))[:data]
+            events = @api_node.read('events', query_read_delete)[:data]
             return { type: :object_list, data: events}
           when :info
             nd_info = @api_node.read('info')[:data]
@@ -331,57 +361,16 @@ module Aspera
           end
         end
 
-        def execute_node_gen4_file_command(command_node_file, top_file_id)
-          file_path = options.get_option(:path)
-          apifid =
-            if file_path.nil?
-              {api: @api_node, file_id: instance_identifier}
-            else
-              @api_node.resolve_api_fid(top_file_id, file_path) # TODO: allow follow link ?
-            end
-          case command_node_file
-          when :show
-            items = apifid[:api].read("files/#{apifid[:file_id]}")[:data]
-            return {type: :single_object, data: items}
-          when :modify
-            update_param = options.get_next_argument('update data', type: Hash)
-            apifid[:api].update("files/#{apifid[:file_id]}", update_param)[:data]
-            return Main.result_status('Done')
-          when :permission
-            command_perm = options.get_next_command(%i[list create delete])
-            case command_perm
-            when :list
-              # generic options : TODO: as arg ? option_url_query
-              list_options ||= {'include' => [Rest::ARRAY_PARAMS, 'access_level', 'permission_count']}
-              # add which one to get
-              list_options['file_id'] = apifid[:file_id]
-              list_options['inherited'] ||= false
-              items = apifid[:api].read('permissions', list_options)[:data]
-              return {type: :object_list, data: items}
-            when :delete
-              perm_id = instance_identifier
-              return do_bulk_operation(perm_id, 'deleted') do |one_id|
-                # TODO: notify event ?
-                apifid[:api].delete("permissions/#{perm_id}")
-                {'id' => one_id}
-              end
-            when :create
-              create_param = options.get_next_argument('creation data', type: Hash)
-              raise 'no file_id' if create_param.key?('file_id')
-              create_param['file_id'] = apifid[:file_id]
-              create_param['access_levels'] = Aspera::Node::ACCESS_LEVELS unless create_param.key?('access_levels')
-              # add application specific tags (AoC)
-              the_app = apifid[:api].app_info
-              the_app[:api].permissions_create_params(create_param: create_param, app_info: the_app) unless the_app.nil?
-              # create permission
-              created_data = apifid[:api].create('permissions', create_param)[:data]
-              # bnotify application of creation
-              the_app[:api].permissions_create_event(created_data: created_data, app_info: the_app) unless the_app.nil?
-              return { type: :single_object, data: created_data}
-            else raise "internal error:shall not reach here (#{command_perm})"
-            end
-          else raise "internal error:shall not reach here (#{command_node_file})"
+        # @return [Hash] api and main file id for given path or id
+        # Allows to specify a file by its path or by its id on the node
+        def apifid_from_next_arg(top_file_id)
+          file_path = instance_identifier(description: 'path or id') do |attribute, value|
+            raise 'Only selection "id" is supported (file id)' unless attribute.eql?('id')
+            # directly return result for method
+            return {api: @api_node, file_id: value}
           end
+          # there was no selector, so it is a path
+          return @api_node.resolve_api_fid(top_file_id, file_path)
         end
 
         def execute_command_gen4(command_repo, top_file_id)
@@ -415,16 +404,16 @@ module Aspera
             apifid = @api_node.resolve_api_fid(top_file_id, options.get_next_argument('path'))
             file_info = apifid[:api].read("files/#{apifid[:file_id]}")[:data]
             if file_info['type'].eql?('folder')
-              result = apifid[:api].read("files/#{apifid[:file_id]}/files", options.get_option(:value))
+              result = apifid[:api].read("files/#{apifid[:file_id]}/files", old_query_read_delete)
               items = result[:data]
-              formatter.display_status("Items: #{result[:data].length}/#{result[:http]['X-Total-Count']}")
+              formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count'])
             else
               items = [file_info]
             end
             return {type: :object_list, data: items, fields: %w[name type recursive_size size modified_time access_level]}
           when :find
             apifid = @api_node.resolve_api_fid(top_file_id, options.get_next_argument('path'))
-            test_block = Aspera::Node.file_matcher(options.get_option(:value))
+            test_block = Aspera::Node.file_matcher(value_or_query(allowed_types: String))
             return {type: :object_list, data: @api_node.find_files(apifid[:file_id], test_block), fields: ['path']}
           when :mkdir
             containing_folder_path = options.get_next_argument('path').split(Aspera::Node::PATH_SEPARATOR)
@@ -434,10 +423,10 @@ module Aspera
             return Main.result_status("created: #{result['name']} (id=#{result['id']})")
           when :rename
             file_path = options.get_next_argument('source path')
-            newname = options.get_next_argument('new name')
             apifid = @api_node.resolve_api_fid(top_file_id, file_path)
+            newname = options.get_next_argument('new name')
             result = apifid[:api].update("files/#{apifid[:file_id]}", {name: newname})[:data]
-            return Main.result_status("renamed #{file_path} to #{newname}")
+            return Main.result_status("renamed to #{newname}")
           when :delete
             return do_bulk_operation(options.get_next_argument('path'), 'deleted', id_result: 'path') do |l_path|
               raise "expecting String (path), got #{l_path.class.name} (#{l_path})" unless l_path.is_a?(String)
@@ -446,12 +435,8 @@ module Aspera
               {'path' => l_path}
             end
           when :sync
-            # remote is specified by option to_folder
-            apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND))
-            transfer_spec = apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)
-            Log.dump(:ts, transfer_spec)
-            sync_plugin = Plugins::Sync.new(@agents, transfer_spec: transfer_spec)
-            return sync_plugin.execute_action
+            node_sync = SyncSpecGen4.new(@api_node, top_file_id)
+            return Plugins::Sync.new(@agents, sync_spec: node_sync).execute_action
           when :upload
             apifid = @api_node.resolve_api_fid(top_file_id, transfer.destination_folder(Fasp::TransferSpec::DIRECTION_SEND))
             return Main.result_transfer(transfer.start(apifid[:api].transfer_spec_gen4(apifid[:file_id], Fasp::TransferSpec::DIRECTION_SEND)))
@@ -498,9 +483,60 @@ module Aspera
               subpath: "files/#{apifid[:file_id]}/content",
               save_to_file: File.join(transfer.destination_folder(Fasp::TransferSpec::DIRECTION_RECEIVE), file_name))
             return Main.result_status("downloaded: #{file_name}")
-          when :file
-            command_node_file = options.get_next_command(%i[show modify permission])
-            return execute_node_gen4_file_command(command_node_file, top_file_id)
+          when :show
+            apifid = apifid_from_next_arg(top_file_id)
+            items = apifid[:api].read("files/#{apifid[:file_id]}")[:data]
+            return {type: :single_object, data: items}
+          when :modify
+            apifid = apifid_from_next_arg(top_file_id)
+            update_param = options.get_next_argument('update data', type: Hash)
+            apifid[:api].update("files/#{apifid[:file_id]}", update_param)[:data]
+            return Main.result_status('Done')
+          when :thumbnail
+            apifid = apifid_from_next_arg(top_file_id)
+            result = apifid[:api].call(
+              operation: 'GET',
+              subpath: "files/#{apifid[:file_id]}/preview",
+              headers: {'Accept' => 'image/png'}
+            )
+            require 'aspera/preview/terminal'
+            return Main.result_status(Preview::Terminal.build(result[:http].body, reserved_lines: 3))
+          when :permission
+            apifid = apifid_from_next_arg(top_file_id)
+            command_perm = options.get_next_command(%i[list create delete])
+            case command_perm
+            when :list
+              # generic options : TODO: as arg ? query_read_delete
+              list_options ||= {'include' => Rest.array_params(%w[access_level permission_count])}
+              # add which one to get
+              list_options['file_id'] = apifid[:file_id]
+              list_options['inherited'] ||= false
+              items = apifid[:api].read('permissions', list_options)[:data]
+              return {type: :object_list, data: items}
+            when :delete
+              perm_id = instance_identifier
+              return do_bulk_operation(perm_id, 'deleted') do |one_id|
+                # TODO: notify event ?
+                apifid[:api].delete("permissions/#{perm_id}")
+                # notify application of deletion
+                the_app[:api].permissions_send_event(created_data: created_data, app_info: the_app, types: ['permission.deleted']) unless the_app.nil?
+                {'id' => one_id}
+              end
+            when :create
+              create_param = options.get_next_argument('creation data', type: Hash)
+              raise 'no file_id' if create_param.key?('file_id')
+              create_param['file_id'] = apifid[:file_id]
+              create_param['access_levels'] = Aspera::Node::ACCESS_LEVELS unless create_param.key?('access_levels')
+              # add application specific tags (AoC)
+              the_app = apifid[:api].app_info
+              the_app[:api].permissions_set_create_params(create_param: create_param, app_info: the_app) unless the_app.nil?
+              # create permission
+              created_data = apifid[:api].create('permissions', create_param)[:data]
+              # notify application of creation
+              the_app[:api].permissions_send_event(created_data: created_data, app_info: the_app) unless the_app.nil?
+              return { type: :single_object, data: created_data}
+            else raise "internal error:shall not reach here (#{command_perm})"
+            end
           else raise "INTERNAL ERROR: no case for #{command_repo}"
           end # command_repo
           # raise 'INTERNAL ERROR: missing return'
@@ -510,65 +546,65 @@ module Aspera
         def execute_async
           command = options.get_next_command(%i[list delete files show counters bandwidth])
           unless command.eql?(:list)
-            asyncname = options.get_option(:sync_name)
-            if asyncname.nil?
-              asyncid = instance_identifier
-              if asyncid.eql?(VAL_ALL) && %i[show delete].include?(command)
-                asyncids = @api_node.read('async/list')[:data]['sync_ids']
+            async_name = options.get_option(:sync_name)
+            if async_name.nil?
+              async_id = instance_identifier
+              if async_id.eql?(VAL_ALL) && %i[show delete].include?(command)
+                async_ids = @api_node.read('async/list')[:data]['sync_ids']
               else
-                Integer(asyncid) # must be integer
-                asyncids = [asyncid]
+                Integer(async_id) # must be integer
+                async_ids = [async_id]
               end
             else
-              asyncids = @api_node.read('async/list')[:data]['sync_ids']
-              summaries = @api_node.create('async/summary', {'syncs' => asyncids})[:data]['sync_summaries']
-              selected = summaries.find{|s|s['name'].eql?(asyncname)}
-              raise "no such sync: #{asyncname}" if selected.nil?
-              asyncid = selected['snid']
-              asyncids = [asyncid]
+              async_ids = @api_node.read('async/list')[:data]['sync_ids']
+              summaries = @api_node.create('async/summary', {'syncs' => async_ids})[:data]['sync_summaries']
+              selected = summaries.find{|s|s['name'].eql?(async_name)}
+              raise "no such sync: #{async_name}" if selected.nil?
+              async_id = selected['snid']
+              async_ids = [async_id]
             end
-            pdata = {'syncs' => asyncids}
+            post_data = {'syncs' => async_ids}
           end
           case command
           when :list
             resp = @api_node.read('async/list')[:data]['sync_ids']
             return { type: :value_list, data: resp, name: 'id' }
           when :show
-            resp = @api_node.create('async/summary', pdata)[:data]['sync_summaries']
+            resp = @api_node.create('async/summary', post_data)[:data]['sync_summaries']
             return Main.result_empty if resp.empty?
-            return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if asyncid.eql?(VAL_ALL)
+            return { type: :object_list, data: resp, fields: %w[snid name local_dir remote_dir] } if async_id.eql?(VAL_ALL)
             return { type: :single_object, data: resp.first }
           when :delete
-            resp = @api_node.create('async/delete', pdata)[:data]
+            resp = @api_node.create('async/delete', post_data)[:data]
             return { type: :single_object, data: resp, name: 'id' }
           when :bandwidth
-            pdata['seconds'] = 100 # TODO: as parameter with --value
-            resp = @api_node.create('async/bandwidth', pdata)[:data]
+            post_data['seconds'] = 100 # TODO: as parameter with --value
+            resp = @api_node.create('async/bandwidth', post_data)[:data]
             data = resp['bandwidth_data']
             return Main.result_empty if data.empty?
-            data = data.first[asyncid]['data']
+            data = data.first[async_id]['data']
             return { type: :object_list, data: data, name: 'id' }
           when :files
             # count int
             # filename str
             # skip int
             # status int
-            filter = options.get_option(:value)
-            pdata.merge!(filter) unless filter.nil?
-            resp = @api_node.create('async/files', pdata)[:data]
+            filter = value_or_query(allowed_types: Hash)
+            post_data.merge!(filter) unless filter.nil?
+            resp = @api_node.create('async/files', post_data)[:data]
             data = resp['sync_files']
-            data = data.first[asyncid] unless data.empty?
+            data = data.first[async_id] unless data.empty?
             iteration_data = []
             skip_ids_persistency = nil
-            if options.get_option(:once_only, is_type: :mandatory)
+            if options.get_option(:once_only, mandatory: true)
               skip_ids_persistency = PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data:    iteration_data,
                 id:      IdGenerator.from_list([
                   'sync_files',
-                  options.get_option(:url, is_type: :mandatory),
-                  options.get_option(:username, is_type: :mandatory),
-                  asyncid]))
+                  options.get_option(:url, mandatory: true),
+                  options.get_option(:username, mandatory: true),
+                  async_id]))
               unless iteration_data.first.nil?
                 data.select!{|l| l['fnid'].to_i > iteration_data.first}
               end
@@ -578,7 +614,7 @@ module Aspera
             skip_ids_persistency&.save
             return { type: :object_list, data: data, name: 'id' }
           when :counters
-            resp = @api_node.create('async/counters', pdata)[:data]['sync_counters'].first[asyncid].last
+            resp = @api_node.create('async/counters', post_data)[:data]['sync_counters'].first[async_id].last
             return Main.result_empty if resp.nil?
             return { type: :single_object, data: resp }
           end
@@ -586,7 +622,7 @@ module Aspera
 
         ACTIONS = %i[
           async
-          sync
+          ssync
           stream
           transfer
           service
@@ -599,18 +635,21 @@ module Aspera
           command ||= options.get_next_command(ACTIONS)
           case command
           when *COMMON_ACTIONS then return execute_simple_common(command, prefix_path)
-          when :async then return execute_async
-          when :sync
-            # newer api
-            sync_command = options.get_next_command(%i[bandwidth counters files start state stop summary].concat(Plugin::ALL_OPS) - %i[modify])
+          when :async then return execute_async # former API
+          when :ssync
+            # newer API
+            sync_command = options.get_next_command(%i[start stop bandwidth counters files state summary].concat(Plugin::ALL_OPS) - %i[modify])
             case sync_command
             when *Plugin::ALL_OPS then return entity_command(sync_command, @api_node, 'asyncs', item_list_key: 'ids')
             else
-              parameters = options.get_option(:value)
               asyncs_id = instance_identifier
+              parameters = nil
               if %i[start stop].include?(sync_command)
                 @api_node.create("asyncs/#{asyncs_id}/#{sync_command}", parameters)
-                return Main.result_status('ok')
+                return Main.result_status('Done')
+              end
+              if %i[bandwidth counters files].include?(sync_command)
+                parameters = value_or_query(allowed_types: Hash, mandatory: false) || {}
               end
               return { type: :single_object, data: @api_node.read("asyncs/#{asyncs_id}/#{sync_command}", parameters)[:data] }
             end
@@ -618,16 +657,16 @@ module Aspera
             command = options.get_next_command(%i[list create show modify cancel])
             case command
             when :list
-              resp = @api_node.read('ops/transfers', options.get_option(:value))
+              resp = @api_node.read('ops/transfers', old_query_read_delete)
               return { type: :object_list, data: resp[:data], fields: %w[id status] } # TODO: useful?
             when :create
-              resp = @api_node.create('streams', options.get_option(:value, is_type: :mandatory))
+              resp = @api_node.create('streams', value_create_modify(command: command, type: Hash))
               return { type: :single_object, data: resp[:data] }
             when :show
               resp = @api_node.read("ops/transfers/#{options.get_next_argument('transfer id')}")
               return { type: :other_struct, data: resp[:data] }
             when :modify
-              resp = @api_node.update("streams/#{options.get_next_argument('transfer id')}", options.get_option(:value, is_type: :mandatory))
+              resp = @api_node.update("streams/#{options.get_next_argument('transfer id')}", value_create_modify(command: command, type: Hash))
               return { type: :other_struct, data: resp[:data] }
             when :cancel
               resp = @api_node.cancel("streams/#{options.get_next_argument('transfer id')}")
@@ -636,21 +675,21 @@ module Aspera
               raise 'error'
             end
           when :transfer
-            command = options.get_next_command(%i[list cancel show])
+            command = options.get_next_command(%i[list cancel show modify bandwidth_average])
             res_class_path = 'ops/transfers'
-            if %i[cancel show].include?(command)
+            if %i[cancel show modify].include?(command)
               one_res_id = instance_identifier
               one_res_path = "#{res_class_path}/#{one_res_id}"
             end
             case command
             when :list
               # could use ? subpath: 'transfers'
-              query = options.get_option(:value) || options.get_option(:query)
+              query = query_read_delete
               raise 'Query must be a Hash' unless query.nil? || query.is_a?(Hash)
-              resp = @api_node.read(res_class_path, query)
+              transfers_data = @api_node.read(res_class_path, query)[:data]
               return {
                 type:   :object_list,
-                data:   resp[:data],
+                data:   transfers_data,
                 fields: %w[id status start_spec.direction start_spec.remote_user start_spec.remote_host start_spec.destination_path]
               }
             when :cancel
@@ -659,13 +698,61 @@ module Aspera
             when :show
               resp = @api_node.read(one_res_path)
               return { type: :other_struct, data: resp[:data] }
+            when :modify
+              resp = @api_node.update(one_res_path, options.get_next_argument('update value', type: Hash))
+              return { type: :other_struct, data: resp[:data] }
+            when :bandwidth_average
+              transfers_data = @api_node.read(res_class_path, query)[:data]
+              # collect all key dates
+              bandwidth_period = {}
+              dir_info = %i[avg_kbps sessions].freeze
+              transfers_data.each do |transfer|
+                session = transfer
+                # transfer['sessions'].each do |session|
+                next if session['avg_rate_kbps'].zero?
+                bandwidth_period[session['start_time_usec']] = 0
+                bandwidth_period[session['end_time_usec']] = 0
+                # end
+              end
+              result = []
+              # all dates sorted numerically
+              all_dates = bandwidth_period.keys.sort
+              all_dates.each_with_index do |start_date, index|
+                end_date = all_dates[index + 1]
+                # do not process last one
+                break if end_date.nil?
+                # init data for this period
+                period_bandwidth = Fasp::TransferSpec::DIRECTION_ENUM_VALUES.map(&:to_sym).each_with_object({}) do |direction, h|
+                  h[direction] = dir_info.each_with_object({}) do |k2, h2|
+                    h2[k2] = 0
+                  end
+                end
+                # find all transfers that were active at this time
+                transfers_data.each do |transfer|
+                  session = transfer
+                  # transfer['sessions'].each do |session|
+                  # skip if not information for this period
+                  next if session['avg_rate_kbps'].zero?
+                  # skip if not in this period
+                  next if session['start_time_usec'] >= end_date || session['end_time_usec'] <= start_date
+                  info = period_bandwidth[transfer['start_spec']['direction'].to_sym]
+                  info[:avg_kbps] += session['avg_rate_kbps']
+                  info[:sessions] += 1
+                  # end
+                end
+                next if Fasp::TransferSpec::DIRECTION_ENUM_VALUES.map(&:to_sym).all? do |dir|
+                  period_bandwidth[dir][:sessions].zero?
+                end
+                result.push({start: Time.at(start_date / 1_000_000), end: Time.at(end_date / 1_000_000)}.merge(period_bandwidth))
+              end
+              return { type: :object_list, data: result }
             else
               raise 'error'
             end
           when :service
             command = options.get_next_command(%i[list create delete])
             if [:delete].include?(command)
-              svcid = instance_identifier
+              service_id = instance_identifier
             end
             case command
             when :list
@@ -677,8 +764,8 @@ module Aspera
               resp = @api_node.create('rund/services', params)
               return Main.result_status("#{resp[:data]['id']} created")
             when :delete
-              @api_node.delete("rund/services/#{svcid}")
-              return Main.result_status("#{svcid} deleted")
+              @api_node.delete("rund/services/#{service_id}")
+              return Main.result_status("#{service_id} deleted")
             end
           when :watch_folder
             res_class_path = 'v3/watchfolders'
@@ -692,15 +779,15 @@ module Aspera
             @api_node.params[:headers]['X-aspera-WF-version'] = '2017_10_23'
             case command
             when :create
-              resp = @api_node.create(res_class_path, options.get_option(:value, is_type: :mandatory))
+              resp = @api_node.create(res_class_path, value_create_modify(command: command, type: Hash))
               return Main.result_status("#{resp[:data]['id']} created")
             when :list
-              resp = @api_node.read(res_class_path, options.get_option(:value))
+              resp = @api_node.read(res_class_path, old_query_read_delete)
               return { type: :value_list, data: resp[:data]['ids'], name: 'id' }
             when :show
               return { type: :single_object, data: @api_node.read(one_res_path)[:data]}
             when :modify
-              @api_node.update(one_res_path, options.get_option(:value, is_type: :mandatory))
+              @api_node.update(one_res_path, value_or_query(mandatory: true, allowed_types: Hash))
               return Main.result_status("#{one_res_id} updated")
             when :delete
               @api_node.delete(one_res_path)
@@ -712,8 +799,7 @@ module Aspera
             command = options.get_next_command(%i[session file])
             validator_id = options.get_option(:validator)
             validation = {'validator_id' => validator_id} unless validator_id.nil?
-            request_data = options.get_option(:value)
-            request_data ||= {}
+            request_data = value_create_modify(default: {}, type: Hash)
             case command
             when :session
               command = options.get_next_command([:list])
@@ -744,16 +830,16 @@ module Aspera
             end
           when :asperabrowser
             browse_params = {
-              'nodeUser' => options.get_option(:username, is_type: :mandatory),
-              'nodePW'   => options.get_option(:password, is_type: :mandatory),
-              'nodeURL'  => options.get_option(:url, is_type: :mandatory)
+              'nodeUser' => options.get_option(:username, mandatory: true),
+              'nodePW'   => options.get_option(:password, mandatory: true),
+              'nodeURL'  => options.get_option(:url, mandatory: true)
             }
             # encode parameters so that it looks good in url
             encoded_params = Base64.strict_encode64(Zlib::Deflate.deflate(JSON.generate(browse_params))).gsub(/=+$/, '').tr('+/', '-_').reverse
             OpenApplication.instance.uri(options.get_option(:asperabrowserurl) + '?goto=' + encoded_params)
             return Main.result_status('done')
           when :basic_token
-            return Main.result_status(Rest.basic_creds(options.get_option(:username, is_type: :mandatory), options.get_option(:password, is_type: :mandatory)))
+            return Main.result_status(Rest.basic_creds(options.get_option(:username, mandatory: true), options.get_option(:password, mandatory: true)))
           end # case command
           raise 'ERROR: shall not reach this line'
         end # execute_action