aspera-cli 4.2.0 → 4.4.0

data/lib/aspera/cli/plugins/faspex.rb

@@ -7,8 +7,10 @@ require 'aspera/persistency_action_once'
 require 'aspera/open_application'
 require 'aspera/fasp/uri'
 require 'aspera/nagios'
+require 'aspera/id_generator'
 require 'xmlsimple'
 require 'json'
+require 'cgi'
 
 module Aspera
   module Cli
@@ -17,7 +19,19 @@ module Aspera
         KEY_NODE='node'
         KEY_PATH='path'
         VAL_ALL='ALL'
-        private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL
+        # added field in result that identifies the package
+        PACKAGE_MATCH_FIELD='package_id'
+        # list of supported atoms
+        ATOM_MAILBOXES=[:inbox, :archive, :sent]
+        # allowed parameters for inbox.atom
+        ATOM_PARAMS=['page', 'count', 'startIndex']
+        # with special parameters (from Plugin class)
+        ATOM_EXT_PARAMS=ATOM_PARAMS+[MAX_ITEMS, MAX_PAGES]
+        # sub path in url for public link delivery
+        PUB_LINK_EXTERNAL_MATCH='external_deliveries/'
+        private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL,:PACKAGE_MATCH_FIELD,:ATOM_MAILBOXES,
+        :ATOM_PARAMS,:ATOM_EXT_PARAMS,:PUB_LINK_EXTERNAL_MATCH
+
         def initialize(env)
           @api_v3=nil
           @api_v4=nil
@@ -26,7 +40,8 @@ module Aspera
           self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
           self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
           self.options.add_opt_simple(:storage,'Faspex local storage definition')
-          self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
+          self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
+          self.options.add_opt_list(:box,ATOM_MAILBOXES,'package box')
           self.options.set_option(:box,:inbox)
           self.options.parse_options!
         end
@@ -101,29 +116,57 @@ module Aspera
           return @api_v4
         end
 
-        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
-
-        # we match recv command on atom feed on this field
-        PACKAGE_MATCH_FIELD='package_id'
-
+        # query supports : {"startIndex":10,"count":1,"page":109}
         def mailbox_all_entries
-          my_user_name=self.options.get_option(:username,:mandatory)
+          recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
+          # mailbox is in ATOM_MAILBOXES
           mailbox=self.options.get_option(:box,:mandatory)
-          all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
-          all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
-          Log.dump(:all_inbox_data,all_inbox_data)
-          result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
-          result.each do |e|
-            case mailbox
-            when :inbox,:archive
-              recipient=e['to'].select{|i|i['name'].first.eql?(my_user_name)}.first
-              e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
-            when :sent
-              e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
+          # parameters
+          mailbox_query=self.options.get_option(:query,:optional)
+          max_items=nil
+          max_pages=nil
+          result=[]
+          if !mailbox_query.nil?
+            raise "query: must be Hash or nil" unless mailbox_query.is_a?(Hash)
+            raise "query: supported params: #{ATOM_EXT_PARAMS}" unless (mailbox_query.keys-ATOM_EXT_PARAMS).empty?
+            raise "query: startIndex and page are exclusive" if mailbox_query.has_key?('startIndex') and mailbox_query.has_key?('page')
+            max_items=mailbox_query[MAX_ITEMS]
+            mailbox_query.delete(MAX_ITEMS)
+            max_pages=mailbox_query[MAX_PAGES]
+            mailbox_query.delete(MAX_PAGES)
+          end
+          loop do
+            atom_xml=api_v3.call({operation: 'GET',subpath: "#{mailbox}.atom",headers: {'Accept'=>'application/xml'},url_params: mailbox_query})[:http].body
+            box_data=XmlSimple.xml_in(atom_xml, {'ForceArray' => true})
+            Log.dump(:box_data,box_data)
+            items=box_data.has_key?('entry') ? box_data['entry'] : []
+            Log.log.debug("new items: #{items.count}")
+            # it is the end if page is empty
+            break if items.empty?
+            items.each do |package|
+              package[PACKAGE_MATCH_FIELD]=case mailbox
+              when :inbox,:archive
+                recipient=package['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
+                recipient.nil? ? nil : recipient['recipient_delivery_id'].first
+              else # :sent
+                package['delivery_id'].first
+              end
+              # keep only those for the specified recipient
+              result.push(package) unless package[PACKAGE_MATCH_FIELD].nil?
             end
+            Log.log.debug("total items: #{result.count}")
+            # reach the limit ?
+            break if !max_items.nil? and result.count > max_items
+            link=box_data['link'].select{|i|i['rel'].eql?('next')}.first
+            Log.log.debug("link: #{link}")
+            # no next link
+            break if link.nil?
+            # replace parameters with the ones from next link
+            params=CGI.parse(URI.parse(link['href']).query)
+            mailbox_query=params.keys.inject({}){|m,i|;m[i]=params[i].first;m}
+            Log.log.debug("query: #{mailbox_query}")
+            break if !max_pages.nil? and mailbox_query['page'].to_i > max_pages
           end
-          # remove dropbox packages
-          result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
           return result
         end
 
@@ -159,6 +202,8 @@ module Aspera
           return pkgdatares.first
         end
 
+        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
+
         def execute_action
           command=self.options.get_next_command(ACTIONS)
           case command
@@ -208,52 +253,59 @@ module Aspera
               #Log.dump('transfer_spec',transfer_spec)
               return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
             when :recv
-              public_link_url=self.options.get_option(:link,:optional)
-              if !public_link_url.nil?
-                link_data=self.class.get_link_data(public_link_url)
-                if !link_data[:subpath].match(%r{external_deliveries/})
-                  raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
+              link_url=self.options.get_option(:link,:optional)
+              # list of faspex ID/URI to download
+              pkg_id_uri=nil
+              skip_ids_data=[]
+              skip_ids_persistency=nil
+              case link_url
+              when nil # usual case: no link
+                if self.options.get_option(:once_only,:mandatory)
+                  skip_ids_persistency=PersistencyActionOnce.new(
+                  manager: @agents[:persistency],
+                  data:    skip_ids_data,
+                  id:      IdGenerator.from_list(['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s]))
+                end
+                # get command line parameters
+                delivid=self.options.get_option(:id,:mandatory)
+                if delivid.eql?(VAL_ALL)
+                  pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
+                  # TODO : remove ids from skip not present in inbox
+                  # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
+                  pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
+                else
+                  recipient=options.get_option(:recipient,:optional)
+                  if !recipient.nil? and recipient.start_with?('*')
+                    raise "Dropbox and Workgroup packages should use link option with faspe:"
+                  end
+                  # TODO: delivery id is the right one if package was receive by workgroup
+                  endpoint=case self.options.get_option(:box,:mandatory)
+                  when :inbox,:archive;'received'
+                  when :sent; 'sent'
+                  end
+                  entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
+                  package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
+                  pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
+                end
+              when /^faspe:/
+                pkg_id_uri=[{:id=>'package',:uri=>link_url}]
+              else
+                link_data=self.class.get_link_data(link_url)
+                if !link_data[:subpath].start_with?(PUB_LINK_EXTERNAL_MATCH)
+                  raise CliBadArgument,"Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
                 end
-                # Note: unauthenticated API
+                # Note: unauthenticated API (autorization is in url params)
                 api_public_link=Rest.new({:base_url=>link_data[:base_url]})
                 pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
                 if !pkgdatares[:http].body.start_with?('<?xml ')
-                  OpenApplication.instance.uri(public_link_url)
+                  OpenApplication.instance.uri(link_url)
                   raise CliError, 'no such package'
                 end
                 package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
+                Log.dump(:package_entry,package_entry)
                 transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
-                transfer_spec=Fasp::Uri.new(transfer_uri).transfer_spec
-                transfer_spec['direction']='receive'
-                return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
+                pkg_id_uri=[{:id=>package_entry['id'],:uri=>transfer_uri}]
               end # public link
-              # get command line parameters
-              delivid=self.options.get_option(:id,:mandatory)
-              # list of faspex ID/URI to download
-              pkg_id_uri=nil
-              skip_ids_data=[]
-              skip_ids_persistency=nil
-              if self.options.get_option(:once_only,:mandatory)
-                skip_ids_persistency=PersistencyActionOnce.new(
-                manager: @agents[:persistency],
-                data: skip_ids_data,
-                ids:  ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
-              end
-              if delivid.eql?(VAL_ALL)
-                pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
-                # TODO : remove ids from skip not present in inbox
-                # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
-                pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
-              else
-                # TODO: delivery id is the right one if package was receive by group
-                endpoint=case self.options.get_option(:box,:mandatory)
-                when :inbox,:archive;'received'
-                when :sent; 'sent'
-                end
-                entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
-                package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
-                pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
-              end
               Log.dump(:pkg_id_uri,pkg_id_uri)
               return Main.result_status('no package') if pkg_id_uri.empty?
               result_transfer=[]
@@ -262,20 +314,17 @@ module Aspera
                 # NOTE: only external users have token in faspe: link !
                 if !transfer_spec.has_key?('token')
                   sanitized=id_uri[:uri].gsub('&','&amp;')
-                  # TODO: file jira
-                  #XXsanitized.gsub!(/%3D%3D$/,'==')
-                  #XXsanitized.gsub!(/%3D$/,'=')
                   xmlpayload='<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="'+sanitized+'"/></url-list>'
                   transfer_spec['token']=api_v3.call({:operation=>'POST',:subpath=>'issue-token?direction=down',:headers=>{'Accept'=>'text/plain','Content-Type'=>'application/vnd.aspera.url-list+xml'},:text_body_params=>xmlpayload})[:http].body
                 end
                 transfer_spec['direction']='receive'
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
-                result_transfer.push({'package'=>id_uri[:id],'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>id_uri[:id],Main::STATUS_FIELD=>statuses})
                 # skip only if all sessions completed
                 skip_ids_data.push(id_uri[:id]) if TransferAgent.session_status(statuses).eql?(:success)
               end
               skip_ids_persistency.save unless skip_ids_persistency.nil?
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             end
           when :source
             command_source=self.options.get_next_command([ :list, :id, :name ])

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,5 +1,6 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'securerandom'
 
 module Aspera
@@ -17,14 +18,20 @@ module Aspera
           options.set_option(:auth,:jwt)
           options.parse_options!
         end
-        ACTIONS=[ :node, :package, :auth_client ]
 
         def set_api
-          faxpex5_api_base_url=options.get_option(:url,:mandatory)
-          faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
-          faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
+          @faxpex5_api_base_url=options.get_option(:url,:mandatory)
+          faxpex5_api_v5_url="#{@faxpex5_api_base_url}/api/v5"
+          faxpex5_api_auth_url="#{@faxpex5_api_base_url}/auth"
           case options.get_option(:auth,:mandatory)
+          when :boot
+            # the password here is the token copied directly from browser in developer mode
+            @api_v5=Rest.new({
+              :base_url => faxpex5_api_v5_url,
+              :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
+            })
           when :web
+            # opens a browser and ask user to auth using web
             @api_v5=Rest.new({
               :base_url => faxpex5_api_v5_url,
               :auth     => {
@@ -34,18 +41,9 @@ module Aspera
               :state          => SecureRandom.uuid,
               :client_id      => options.get_option(:client_id,:mandatory),
               :redirect_uri   => options.get_option(:redirect_uri,:mandatory),
-              #:token_field    =>'auth_token',
-              #:path_token     => 'token',
-              #:path_authorize => 'authorize',
-              #:userpass_body  => {name: faxpex5_username,password: faxpex5_password}
               }})
-          when :boot
-            @api_v5=Rest.new({
-              :base_url => faxpex5_api_v5_url,
-              :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
-            })
           when :jwt
-            #raise "JWT to be implemented"
+            # currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
             app_client_id=options.get_option(:client_id,:mandatory)
             @api_v5=Rest.new({
               :base_url => faxpex5_api_v5_url,
@@ -53,28 +51,33 @@ module Aspera
               :type                => :oauth2,
               :base_url            => faxpex5_api_auth_url,
               :grant               => :jwt,
+              :f5_username         => options.get_option(:username,:mandatory),
+              :f5_password         => options.get_option(:password,:mandatory),
               :client_id           => app_client_id,
               :client_secret       => options.get_option(:client_secret,:mandatory),
-              #:redirect_uri   => options.get_option(:redirect_uri,:mandatory),
               :jwt_subject         => "client:#{app_client_id}", # TODO Mmmm
               :jwt_audience        => app_client_id, # TODO Mmmm
               :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
-              :jwt_is_f5           => true,
               :jwt_headers         => {typ: 'JWT'}
               }})
           end
         end
 
+        ACTIONS=[ :node, :package, :auth_client, :jobs ]
+
         #
         def execute_action
           set_api
           command=options.get_next_command(ACTIONS)
           case command
           when :auth_client
-            api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
+            api_auth=Rest.new(@api_v5.params.merge({base_url: "#{@faxpex5_api_base_url}/auth"}))
             return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
           when :node
             return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
+          when :jobs
+            # to test JWT
+            return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
           when :package
             command=options.get_next_command([:list,:show,:send,:receive])
             case command
@@ -86,7 +89,7 @@ module Aspera
               return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
             when :send
               parameters=options.get_option(:value,:mandatory)
-              raise CliBadArgument,'package value must be hash, refer to API' unless parameters.is_a?(Hash)
+              raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
               package=@api_v5.create('packages',parameters)[:data]
               transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
               transfer_spec.delete('authentication')
@@ -98,16 +101,17 @@ module Aspera
               skip_ids_data=[]
               skip_ids_persistency=nil
               if options.get_option(:once_only,:mandatory)
+                # read ids from persistency
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
-                data: skip_ids_data,
-                ids:  ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
+                data:    skip_ids_data,
+                id:      IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
               end
               if pack_id.eql?(VAL_ALL)
-                # todo: if packages have same name, they will overwrite
+                # TODO: if packages have same name, they will overwrite
                 parameters=options.get_option(:value,:optional)
-                parameters||={"type"=>"received","subtype"=>"mypackages","limit"=>1000}
-                raise CliBadArgument,'value filter must be hash (API GET)' unless parameters.is_a?(Hash)
+                parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
+                raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
                 package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
                 package_ids.select!{|i|!skip_ids_data.include?(i)}
               end
@@ -117,12 +121,12 @@ module Aspera
                 transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
                 transfer_spec.delete('authentication')
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
-                result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>id,Main::STATUS_FIELD=>statuses})
                 # skip only if all sessions completed
                 skip_ids_data.push(id) if TransferAgent.session_status(statuses).eql?(:success)
               end
               skip_ids_persistency.save unless skip_ids_persistency.nil?
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             end
           end
         end

data/lib/aspera/cli/plugins/node.rb

@@ -1,6 +1,7 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/nagios'
 require 'aspera/hash_ext'
+require 'aspera/id_generator'
 require 'base64'
 require 'zlib'
 
@@ -12,22 +13,34 @@ module Aspera
         private_constant :SAMPLE_SOAP_CALL
         def initialize(env)
           super(env)
-          # this is added to some requests , for instance to add tags
+          # this is added to some requests , for instance to add tags (COS)
           @add_request_param = env[:add_request_param] || {}
           unless env[:skip_basic_auth_options]
             self.options.add_opt_simple(:validator,"identifier of validator (optional for central)")
             self.options.add_opt_simple(:asperabrowserurl,"URL for simple aspera web ui")
             self.options.add_opt_simple(:name,"sync name")
-            self.options.add_opt_list(:token,[:aspera,:basic,:auto],'todo: type of token used for transfers')
+            self.options.add_opt_list(:token_type,[:aspera,:basic,:hybrid],'Type of token used for transfers')
             self.options.set_option(:asperabrowserurl,'https://asperabrowser.mybluemix.net')
-            self.options.set_option(:token,:aspera)
+            self.options.set_option(:token_type,:aspera)
             self.options.parse_options!
           end
           return if env[:man_only]
           if env.has_key?(:node_api)
             @api_node=env[:node_api]
           else
-            @api_node=basic_auth_api unless env[:man_only]
+            if self.options.get_option(:password,:mandatory).start_with?('Bearer ')
+              # info is provided like node_info of aoc
+              @api_node=Rest.new({
+                base_url: self.options.get_option(:url,:mandatory),
+                headers: {
+                'Authorization'      => self.options.get_option(:password,:mandatory),
+                'X-Aspera-AccessKey' => self.options.get_option(:username,:mandatory),
+                }
+              })
+            else
+              # this is normal case
+              @api_node=basic_auth_api
+            end
           end
         end
 
@@ -190,22 +203,36 @@ module Aspera
               #raise "unknown type: #{send_result['self']['type']}"
             end
             return c_result_remove_prefix_path(result,'path',prefix_path)
-          when :upload
-            # we send only a list of one transfer request
-            transfer_request = { :paths => [ { :destination => self.transfer.destination_folder('send') } ] }
-            transfer_request.deep_merge!(@add_request_param)
-            send_result=@api_node.create('files/upload_setup',{:transfer_requests => [ { :transfer_request => transfer_request } ] } )[:data]
-            # only one request, so only one answer
-            transfer_spec=send_result['transfer_specs'].first['transfer_spec']
-            # delete this part, as the returned value contains only destination, and not sources
-            transfer_spec.delete('paths')
-            return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
-          when :download
-            transfer_request = {:paths => self.transfer.ts_source_paths }
-            transfer_request.deep_merge!(@add_request_param)
-            send_result=@api_node.create('files/download_setup',{:transfer_requests => [ { :transfer_request => transfer_request } ] } )[:data]
-            # only one request, so only one answer
-            transfer_spec=send_result['transfer_specs'].first['transfer_spec']
+          when :upload,:download
+            token_type=self.options.get_option(:token_type,:optional)
+            # nil if Shares 1.x
+            token_type=:aspera if token_type.nil?
+            case token_type
+            when :aspera,:hybrid
+              transfer_paths=case command
+              when :upload;[ { :destination => self.transfer.destination_folder('send') } ]
+              when :download;self.transfer.ts_source_paths
+              end
+              # only one request, so only one answer
+              transfer_spec=@api_node.create("files/#{command}_setup",{:transfer_requests => [ { transfer_request: {
+                paths: transfer_paths
+                }.deep_merge(@add_request_param) } ] } )[:data]['transfer_specs'].first['transfer_spec']
+              # delete this part, as the returned value contains only destination, and not sources
+              transfer_spec.delete('paths') if command.eql?(:upload)
+            when :basic
+              raise "shall have auth" unless @api_node.params[:auth].is_a?(Hash)
+              raise "shall be basic auth" unless @api_node.params[:auth][:type].eql?(:basic)
+              transfer_spec={
+                'remote_host'=>URI.parse(@api_node.params[:base_url]).host,
+                'remote_user'=>Aspera::Node::ACCESS_KEY_TRANSFER_USER,
+                'ssh_port'=>Aspera::Node::SSH_PORT_DEFAULT,
+                'direction'=>case command;when :upload;'send';when :download;'recv';else raise "Error";end
+              }
+            else raise "ERROR: token_type #{tt}"
+            end
+            if [:basic,:hybrid].include?(token_type)
+              Aspera::Node.set_ak_basic_token(transfer_spec,@api_node.params[:auth][:username],@api_node.params[:auth][:password])
+            end
             return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
           when :api_details
             return { :type=>:single_object, :data => @api_node.params }
@@ -271,8 +298,8 @@ module Aspera
             if self.options.get_option(:once_only,:mandatory)
               skip_ids_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
-              data: iteration_data,
-              ids:  ['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid])
+              data:    iteration_data,
+              id:      IdGenerator.from_list(['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid]))
               unless iteration_data.first.nil?
                 data.select!{|l| l['fnid'].to_i>iteration_data.first}
               end
@@ -408,9 +435,11 @@ module Aspera
               command=self.options.get_next_command([ :list, :modify])
               case command
               when :list
-                request_data.deep_merge!({"validation"=>validation}) unless validation.nil?
-                resp=@api_node.create('services/rest/transfers/v1/files',request_data)
-                return {:type=>:object_list,:data=>resp[:data]["file_transfer_info_result"]["file_transfer_info"],:fields=>["session_uuid","file_id","status","path"]}
+                request_data.deep_merge!({'validation'=>validation}) unless validation.nil?
+                resp=@api_node.create('services/rest/transfers/v1/files',request_data)[:data]
+                resp=JSON.parse(resp) if resp.is_a?(String)
+                Log.dump(:resp,resp)
+                return {:type=>:object_list,:data=>resp['file_transfer_info_result']['file_transfer_info'],:fields=>["session_uuid","file_id","status","path"]}
               when :modify
                 request_data.deep_merge!(validation) unless validation.nil?
                 @api_node.update('services/rest/transfers/v1/files',request_data)