aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/cli/plugins/faspex.rb

@@ -7,8 +7,10 @@ require 'aspera/persistency_action_once'
 require 'aspera/open_application'
 require 'aspera/fasp/uri'
 require 'aspera/nagios'
+require 'aspera/id_generator'
 require 'xmlsimple'
 require 'json'
+require 'cgi'
 
 module Aspera
   module Cli
@@ -17,7 +19,33 @@ module Aspera
         KEY_NODE='node'
         KEY_PATH='path'
         VAL_ALL='ALL'
-        private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL
+        # added field in result that identifies the package
+        PACKAGE_MATCH_FIELD='package_id'
+        # list of supported atoms
+        ATOM_MAILBOXES=[:inbox, :archive, :sent]
+        # allowed parameters for inbox.atom
+        ATOM_PARAMS=['page', 'count', 'startIndex']
+        # with special parameters (from Plugin class)
+        ATOM_EXT_PARAMS=ATOM_PARAMS+[MAX_ITEMS, MAX_PAGES]
+        # sub path in url for public link delivery
+        PUB_LINK_EXTERNAL_MATCH='external_deliveries/'
+        private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL,:PACKAGE_MATCH_FIELD,:ATOM_MAILBOXES,
+        :ATOM_PARAMS,:ATOM_EXT_PARAMS,:PUB_LINK_EXTERNAL_MATCH
+
+        class << self
+          def detect(base_url)
+            api=Rest.new({:base_url=>base_url})
+            result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
+            # 4.x
+            if result[:http].body.start_with?('<?xml')
+              res_s=XmlSimple.xml_in(result[:http].body, {'ForceArray' => false})
+              version=res_s['XRD']['application']['version']
+              return {version: version}
+            end
+            return nil
+          end
+        end
+
         def initialize(env)
           @api_v3=nil
           @api_v4=nil
@@ -27,7 +55,7 @@ module Aspera
           self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
           self.options.add_opt_simple(:storage,'Faspex local storage definition')
           self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
-          self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
+          self.options.add_opt_list(:box,ATOM_MAILBOXES,'package box')
           self.options.set_option(:box,:inbox)
           self.options.parse_options!
         end
@@ -102,29 +130,64 @@ module Aspera
           return @api_v4
         end
 
-        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
-
-        # we match recv command on atom feed on this field
-        PACKAGE_MATCH_FIELD='package_id'
-
+        # query supports : {"startIndex":10,"count":1,"page":109}
         def mailbox_all_entries
           recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
+          # mailbox is in ATOM_MAILBOXES
           mailbox=self.options.get_option(:box,:mandatory)
-          all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
-          all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
-          Log.dump(:all_inbox_data,all_inbox_data)
-          result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
-          result.each do |e|
-            case mailbox
-            when :inbox,:archive
-              recipient=e['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
-              e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
-            when :sent
-              e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
+          # parameters
+          mailbox_query=self.options.get_option(:query,:optional)
+          max_items=nil
+          max_pages=nil
+          result=[]
+          if !mailbox_query.nil?
+            raise "query: must be Hash or nil" unless mailbox_query.is_a?(Hash)
+            raise "query: supported params: #{ATOM_EXT_PARAMS}" unless (mailbox_query.keys-ATOM_EXT_PARAMS).empty?
+            raise "query: startIndex and page are exclusive" if mailbox_query.has_key?('startIndex') and mailbox_query.has_key?('page')
+            max_items=mailbox_query[MAX_ITEMS]
+            mailbox_query.delete(MAX_ITEMS)
+            max_pages=mailbox_query[MAX_PAGES]
+            mailbox_query.delete(MAX_PAGES)
+          end
+          loop do
+            # get a batch of package information
+            # order: first batch is latest packages, and then in a batch ids are increasing
+            atom_xml=api_v3.call({operation: 'GET',subpath: "#{mailbox}.atom",headers: {'Accept'=>'application/xml'},url_params: mailbox_query})[:http].body
+            box_data=XmlSimple.xml_in(atom_xml, {'ForceArray' => true})
+            Log.dump(:box_data,box_data)
+            items=box_data.has_key?('entry') ? box_data['entry'] : []
+            Log.log.debug("new items: #{items.count}")
+            # it is the end if page is empty
+            break if items.empty?
+            # results will be sorted in reverse id
+            items.reverse.each do |package|
+              package[PACKAGE_MATCH_FIELD]=case mailbox
+              when :inbox,:archive
+                recipient=package['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
+                recipient.nil? ? nil : recipient['recipient_delivery_id'].first
+              else # :sent
+                package['delivery_id'].first
+              end
+              # keep only those for the specified recipient,
+              result.push(package) unless package[PACKAGE_MATCH_FIELD].nil?
             end
+            #result.push({PACKAGE_MATCH_FIELD=>'======'})
+            Log.log.debug("total items: #{result.count}")
+            # reach the limit ?
+            if !max_items.nil? and result.count >= max_items
+              result=result.slice(0,max_items) if result.count > max_items
+              break
+            end
+            link=box_data['link'].select{|i|i['rel'].eql?('next')}.first
+            Log.log.debug("link: #{link}")
+            # no next link
+            break if link.nil?
+            # replace parameters with the ones from next link
+            params=CGI.parse(URI.parse(link['href']).query)
+            mailbox_query=params.keys.inject({}){|m,i|;m[i]=params[i].first;m}
+            Log.log.debug("query: #{mailbox_query}")
+            break if !max_pages.nil? and mailbox_query['page'].to_i > max_pages
           end
-          # remove dropbox packages
-          result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
           return result
         end
 
@@ -160,6 +223,8 @@ module Aspera
           return pkgdatares.first
         end
 
+        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
+
         def execute_action
           command=self.options.get_next_command(ACTIONS)
           case command
@@ -215,44 +280,27 @@ module Aspera
               skip_ids_data=[]
               skip_ids_persistency=nil
               case link_url
-              when nil
-                # usual case: no link
-              when /^faspe:/
-                pkg_id_uri=[{:id=>'package',:uri=>link_url}]
-              else
-                link_data=self.class.get_link_data(link_url)
-                if !link_data[:subpath].match(%r{external_deliveries/})
-                  raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
-                end
-                # Note: unauthenticated API
-                api_public_link=Rest.new({:base_url=>link_data[:base_url]})
-                pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
-                if !pkgdatares[:http].body.start_with?('<?xml ')
-                  OpenApplication.instance.uri(link_url)
-                  raise CliError, 'no such package'
-                end
-                package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
-                transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
-                transfer_spec=Fasp::Uri.new(transfer_uri).transfer_spec
-                transfer_spec['direction']='receive'
-                return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
-              end # public link
-              if pkg_id_uri.nil?
-                # get command line parameters
-                delivid=self.options.get_option(:id,:mandatory)
+              when nil # usual case: no link
                 if self.options.get_option(:once_only,:mandatory)
                   skip_ids_persistency=PersistencyActionOnce.new(
                   manager: @agents[:persistency],
-                  data: skip_ids_data,
-                  ids:  ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
+                  data:    skip_ids_data,
+                  id:      IdGenerator.from_list(['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s]))
                 end
+                # get command line parameters
+                delivid=self.instance_identifier()
+                raise "empty id" if delivid.empty?
                 if delivid.eql?(VAL_ALL)
                   pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
-                  # TODO : remove ids from skip not present in inbox
+                  # TODO : remove ids from skip not present in inbox to avoid growing too big
                   # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
                   pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
                 else
-                  # TODO: delivery id is the right one if package was receive by group
+                  recipient=options.get_option(:recipient,:optional)
+                  if !recipient.nil? and recipient.start_with?('*')
+                    raise "Dropbox and Workgroup packages should use link option with faspe:"
+                  end
+                  # TODO: delivery id is the right one if package was receive by workgroup
                   endpoint=case self.options.get_option(:box,:mandatory)
                   when :inbox,:archive;'received'
                   when :sent; 'sent'
@@ -261,7 +309,25 @@ module Aspera
                   package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
                   pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
                 end
-              end
+              when /^faspe:/
+                pkg_id_uri=[{:id=>'package',:uri=>link_url}]
+              else
+                link_data=self.class.get_link_data(link_url)
+                if !link_data[:subpath].start_with?(PUB_LINK_EXTERNAL_MATCH)
+                  raise CliBadArgument,"Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
+                end
+                # Note: unauthenticated API (authorization is in url params)
+                api_public_link=Rest.new({:base_url=>link_data[:base_url]})
+                pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
+                if !pkgdatares[:http].body.start_with?('<?xml ')
+                  OpenApplication.instance.uri(link_url)
+                  raise CliError, 'no such package'
+                end
+                package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
+                Log.dump(:package_entry,package_entry)
+                transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
+                pkg_id_uri=[{:id=>package_entry['id'],:uri=>transfer_uri}]
+              end # public link
               Log.dump(:pkg_id_uri,pkg_id_uri)
               return Main.result_status('no package') if pkg_id_uri.empty?
               result_transfer=[]
@@ -270,20 +336,17 @@ module Aspera
                 # NOTE: only external users have token in faspe: link !
                 if !transfer_spec.has_key?('token')
                   sanitized=id_uri[:uri].gsub('&','&amp;')
-                  # TODO: file jira
-                  #XXsanitized.gsub!(/%3D%3D$/,'==')
-                  #XXsanitized.gsub!(/%3D$/,'=')
                   xmlpayload='<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="'+sanitized+'"/></url-list>'
                   transfer_spec['token']=api_v3.call({:operation=>'POST',:subpath=>'issue-token?direction=down',:headers=>{'Accept'=>'text/plain','Content-Type'=>'application/vnd.aspera.url-list+xml'},:text_body_params=>xmlpayload})[:http].body
                 end
                 transfer_spec['direction']='receive'
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
-                result_transfer.push({'package'=>id_uri[:id],'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>id_uri[:id],Main::STATUS_FIELD=>statuses})
                 # skip only if all sessions completed
                 skip_ids_data.push(id_uri[:id]) if TransferAgent.session_status(statuses).eql?(:success)
               end
               skip_ids_persistency.save unless skip_ids_persistency.nil?
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             end
           when :source
             command_source=self.options.get_next_command([ :list, :id, :name ])
@@ -359,7 +422,7 @@ module Aspera
               return self.entity_action(api_v4,'metadata_profiles',nil,:id)
             when :package
               pkg_box_type=self.options.get_next_command([:users])
-              pkg_box_id=self.options.get_option(:id,:mandatory)
+              pkg_box_id=self.instance_identifier()
               return self.entity_action(api_v4,"#{pkg_box_type}/#{pkg_box_id}/packages",nil,:id)
             end
           when :address_book

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,27 +1,41 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'securerandom'
 
 module Aspera
   module Cli
     module Plugins
       class Faspex5 < BasicAuthPlugin
+        class << self
+          def detect(base_url)
+            api=Rest.new({:base_url=>base_url})
+            result=api.read('api/v5/configuration/ping')
+            if result[:http].code.start_with?('2') and result[:http].body.strip.empty?
+              return {version: '5'}
+            end
+            return nil
+          end
+        end
+
         VAL_ALL='ALL'
+        private_constant :VAL_ALL
+
         def initialize(env)
           super(env)
-          options.add_opt_simple(:client_id,'API client identifier in application')
-          options.add_opt_simple(:client_secret,'API client secret in application')
-          options.add_opt_simple(:redirect_uri,'API client redirect URI')
-          options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
-          options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
+          options.add_opt_simple(:client_id,'OAuth client identifier')
+          options.add_opt_simple(:client_secret,'OAuth client secret')
+          options.add_opt_simple(:redirect_uri,'OAuth redirect URI')
+          options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'OAuth type of authentication')
+          options.add_opt_simple(:private_key,'Oauth RSA private key PEM value for JWT (prefix file path with @val:@file:)')
           options.set_option(:auth,:jwt)
           options.parse_options!
         end
 
         def set_api
-          faxpex5_api_base_url=options.get_option(:url,:mandatory)
-          faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
-          faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
+          @faxpex5_api_base_url=options.get_option(:url,:mandatory)
+          faxpex5_api_v5_url="#{@faxpex5_api_base_url}/api/v5"
+          faxpex5_api_auth_url="#{@faxpex5_api_base_url}/auth"
           case options.get_option(:auth,:mandatory)
           when :boot
             # the password here is the token copied directly from browser in developer mode
@@ -42,7 +56,7 @@ module Aspera
               :redirect_uri   => options.get_option(:redirect_uri,:mandatory),
               }})
           when :jwt
-            # currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
+            # currently Faspex 5 beta 4 only supports non-user based apis (e.g. jobs)
             app_client_id=options.get_option(:client_id,:mandatory)
             @api_v5=Rest.new({
               :base_url => faxpex5_api_v5_url,
@@ -50,12 +64,13 @@ module Aspera
               :type                => :oauth2,
               :base_url            => faxpex5_api_auth_url,
               :grant               => :jwt,
+              :f5_username         => options.get_option(:username,:mandatory),
+              :f5_password         => options.get_option(:password,:mandatory),
               :client_id           => app_client_id,
               :client_secret       => options.get_option(:client_secret,:mandatory),
               :jwt_subject         => "client:#{app_client_id}", # TODO Mmmm
               :jwt_audience        => app_client_id, # TODO Mmmm
               :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
-              :jwt_is_f5           => true, # TODO: remove when clarified
               :jwt_headers         => {typ: 'JWT'}
               }})
           end
@@ -69,7 +84,7 @@ module Aspera
           command=options.get_next_command(ACTIONS)
           case command
           when :auth_client
-            api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
+            api_auth=Rest.new(@api_v5.params.merge({base_url: "#{@faxpex5_api_base_url}/auth"}))
             return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
           when :node
             return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
@@ -83,7 +98,7 @@ module Aspera
               parameters=options.get_option(:value,:optional)
               return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
             when :show
-              id=options.get_option(:id,:mandatory)
+              id=instance_identifier()
               return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
             when :send
               parameters=options.get_option(:value,:mandatory)
@@ -94,7 +109,7 @@ module Aspera
               return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
             when :receive
               pkg_type='received'
-              pack_id=options.get_option(:id,:mandatory)
+              pack_id=instance_identifier()
               package_ids=[pack_id]
               skip_ids_data=[]
               skip_ids_persistency=nil
@@ -102,8 +117,8 @@ module Aspera
                 # read ids from persistency
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
-                data: skip_ids_data,
-                ids:  ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
+                data:    skip_ids_data,
+                id:      IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
               end
               if pack_id.eql?(VAL_ALL)
                 # TODO: if packages have same name, they will overwrite
@@ -119,12 +134,12 @@ module Aspera
                 transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
                 transfer_spec.delete('authentication')
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
-                result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>id,Main::STATUS_FIELD=>statuses})
                 # skip only if all sessions completed
                 skip_ids_data.push(id) if TransferAgent.session_status(statuses).eql?(:success)
               end
               skip_ids_persistency.save unless skip_ids_persistency.nil?
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             end
           end
         end

data/lib/aspera/cli/plugins/node.rb

@@ -1,6 +1,8 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/nagios'
 require 'aspera/hash_ext'
+require 'aspera/id_generator'
+require 'aspera/node'
 require 'base64'
 require 'zlib'
 
@@ -8,26 +10,49 @@ module Aspera
   module Cli
     module Plugins
       class Node < BasicAuthPlugin
+        class << self
+          def detect(base_url)
+            api=Rest.new({:base_url=>base_url})
+            result=api.call({:operation=>'GET',:subpath=>'ping'})
+            if result[:http].body.eql?('')
+              return {:product=>:node,:version=>'unknown'}
+            end
+            return nil
+          end
+        end
         SAMPLE_SOAP_CALL='<?xml version="1.0" encoding="UTF-8"?><soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:typ="urn:Aspera:XML:FASPSessionNET:2009/11:Types"><soapenv:Header></soapenv:Header><soapenv:Body><typ:GetSessionInfoRequest><SessionFilter><SessionStatus>running</SessionStatus></SessionFilter></typ:GetSessionInfoRequest></soapenv:Body></soapenv:Envelope>'
         private_constant :SAMPLE_SOAP_CALL
+
         def initialize(env)
           super(env)
-          # this is added to some requests , for instance to add tags
+          # this is added to some requests , for instance to add tags (COS)
           @add_request_param = env[:add_request_param] || {}
           unless env[:skip_basic_auth_options]
             self.options.add_opt_simple(:validator,"identifier of validator (optional for central)")
             self.options.add_opt_simple(:asperabrowserurl,"URL for simple aspera web ui")
             self.options.add_opt_simple(:name,"sync name")
-            self.options.add_opt_list(:token,[:aspera,:basic,:auto],'todo: type of token used for transfers')
+            self.options.add_opt_list(:token_type,[:aspera,:basic,:hybrid],'Type of token used for transfers')
             self.options.set_option(:asperabrowserurl,'https://asperabrowser.mybluemix.net')
-            self.options.set_option(:token,:aspera)
+            self.options.set_option(:token_type,:aspera)
             self.options.parse_options!
           end
           return if env[:man_only]
           if env.has_key?(:node_api)
             @api_node=env[:node_api]
           else
-            @api_node=basic_auth_api unless env[:man_only]
+            if self.options.get_option(:password,:mandatory).start_with?('Bearer ')
+              # info is provided like node_info of aoc
+              @api_node=Rest.new({
+                base_url: self.options.get_option(:url,:mandatory),
+                headers: {
+                'Authorization'      => self.options.get_option(:password,:mandatory),
+                'X-Aspera-AccessKey' => self.options.get_option(:username,:mandatory),
+                }
+              })
+            else
+              # this is normal case
+              @api_node=basic_auth_api
+            end
           end
         end
 
@@ -152,7 +177,7 @@ module Aspera
           when :mkdir
             path_list=get_next_arg_add_prefix(prefix_path,"folder path or ext.val. list")
             path_list=[path_list] unless path_list.is_a?(Array)
-            #TODO
+            #TODO: a command for that ?
             #resp=@api_node.create('space',{ "paths" => path_list.map {|i| {:type=>:directory,:path=>i} } } )
             resp=@api_node.create('files/create',{ "paths" => [{ :type => :directory, :path => path_list } ] } )
             return c_result_translate_rem_prefix(resp,'folder','created',prefix_path)
@@ -190,22 +215,36 @@ module Aspera
               #raise "unknown type: #{send_result['self']['type']}"
             end
             return c_result_remove_prefix_path(result,'path',prefix_path)
-          when :upload
-            # we send only a list of one transfer request
-            transfer_request = { :paths => [ { :destination => self.transfer.destination_folder('send') } ] }
-            transfer_request.deep_merge!(@add_request_param)
-            send_result=@api_node.create('files/upload_setup',{:transfer_requests => [ { :transfer_request => transfer_request } ] } )[:data]
-            # only one request, so only one answer
-            transfer_spec=send_result['transfer_specs'].first['transfer_spec']
-            # delete this part, as the returned value contains only destination, and not sources
-            transfer_spec.delete('paths')
-            return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
-          when :download
-            transfer_request = {:paths => self.transfer.ts_source_paths }
-            transfer_request.deep_merge!(@add_request_param)
-            send_result=@api_node.create('files/download_setup',{:transfer_requests => [ { :transfer_request => transfer_request } ] } )[:data]
-            # only one request, so only one answer
-            transfer_spec=send_result['transfer_specs'].first['transfer_spec']
+          when :upload,:download
+            token_type=self.options.get_option(:token_type,:optional)
+            # nil if Shares 1.x
+            token_type=:aspera if token_type.nil?
+            case token_type
+            when :aspera,:hybrid
+              transfer_paths=case command
+              when :upload;[ { :destination => self.transfer.destination_folder('send') } ]
+              when :download;self.transfer.ts_source_paths
+              end
+              # only one request, so only one answer
+              transfer_spec=@api_node.create("files/#{command}_setup",{:transfer_requests => [ { transfer_request: {
+                paths: transfer_paths
+                }.deep_merge(@add_request_param) } ] } )[:data]['transfer_specs'].first['transfer_spec']
+              # delete this part, as the returned value contains only destination, and not sources
+              transfer_spec.delete('paths') if command.eql?(:upload)
+            when :basic
+              raise "shall have auth" unless @api_node.params[:auth].is_a?(Hash)
+              raise "shall be basic auth" unless @api_node.params[:auth][:type].eql?(:basic)
+              transfer_spec={
+                'remote_host'=>URI.parse(@api_node.params[:base_url]).host,
+                'remote_user'=>Aspera::Fasp::Default::ACCESS_KEY_TRANSFER_USER,
+                'ssh_port'   =>Aspera::Fasp::Default::SSH_PORT,
+                'direction'  =>case command;when :upload;'send';when :download;'recv';else raise "Error";end
+              }
+            else raise "ERROR: token_type #{tt}"
+            end
+            if [:basic,:hybrid].include?(token_type)
+              Aspera::Node.set_ak_basic_token(transfer_spec,@api_node.params[:auth][:username],@api_node.params[:auth][:password])
+            end
             return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
           when :api_details
             return { :type=>:single_object, :data => @api_node.params }
@@ -217,7 +256,7 @@ module Aspera
           unless command.eql?(:list)
             asyncname=self.options.get_option(:name,:optional)
             if asyncname.nil?
-              asyncid=self.options.get_option(:id,:mandatory)
+              asyncid=self.instance_identifier()
               if asyncid.eql?('ALL') and [:show,:delete].include?(command)
                 asyncids=@api_node.read('async/list')[:data]['sync_ids']
               else
@@ -271,8 +310,8 @@ module Aspera
             if self.options.get_option(:once_only,:mandatory)
               skip_ids_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
-              data: iteration_data,
-              ids:  ['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid])
+              data:    iteration_data,
+              id:      IdGenerator.from_list(['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid]))
               unless iteration_data.first.nil?
                 data.select!{|l| l['fnid'].to_i>iteration_data.first}
               end
@@ -300,7 +339,7 @@ module Aspera
             case command
             when :list
               resp=@api_node.read('ops/transfers',self.options.get_option(:value,:optional))
-              return { :type => :object_list, :data => resp[:data], :fields=>['id','status']  } # TODO
+              return { :type => :object_list, :data => resp[:data], :fields=>['id','status']  } # TODO: useful?
             when :create
               resp=@api_node.create('streams',self.options.get_option(:value,:mandatory))
               return { :type => :single_object, :data => resp[:data] }
@@ -323,7 +362,7 @@ module Aspera
             command=self.options.get_next_command([ :list, :cancel, :show ])
             res_class_path='ops/transfers'
             if [:cancel, :show].include?(command)
-              one_res_id=self.options.get_option(:id,:mandatory)
+              one_res_id=self.instance_identifier()
               one_res_path="#{res_class_path}/#{one_res_id}"
             end
             case command
@@ -345,7 +384,7 @@ module Aspera
           when :service
             command=self.options.get_next_command([ :list, :create, :delete])
             if [:delete].include?(command)
-              svcid=self.options.get_option(:id,:mandatory)
+              svcid=self.instance_identifier()
             end
             case command
             when :list
@@ -365,7 +404,7 @@ module Aspera
             #return entity_action(@api_node,'v3/watchfolders',nil,:id)
             command=self.options.get_next_command([ :create, :list, :show, :modify, :delete, :state])
             if [:show,:modify,:delete,:state].include?(command)
-              one_res_id=self.options.get_option(:id,:mandatory)
+              one_res_id=self.instance_identifier()
               one_res_path="#{res_class_path}/#{one_res_id}"
             end
             # hum, to avoid: Unable to convert 2016_09_14 configuration
@@ -408,9 +447,11 @@ module Aspera
               command=self.options.get_next_command([ :list, :modify])
               case command
               when :list
-                request_data.deep_merge!({"validation"=>validation}) unless validation.nil?
-                resp=@api_node.create('services/rest/transfers/v1/files',request_data)
-                return {:type=>:object_list,:data=>resp[:data]["file_transfer_info_result"]["file_transfer_info"],:fields=>["session_uuid","file_id","status","path"]}
+                request_data.deep_merge!({'validation'=>validation}) unless validation.nil?
+                resp=@api_node.create('services/rest/transfers/v1/files',request_data)[:data]
+                resp=JSON.parse(resp) if resp.is_a?(String)
+                Log.dump(:resp,resp)
+                return {:type=>:object_list,:data=>resp['file_transfer_info_result']['file_transfer_info'],:fields=>["session_uuid","file_id","status","path"]}
               when :modify
                 request_data.deep_merge!(validation) unless validation.nil?
                 @api_node.update('services/rest/transfers/v1/files',request_data)

data/lib/aspera/cli/plugins/orchestrator.rb

@@ -110,12 +110,14 @@ module Aspera
             return {:type=>:object_list,:data=>result['Plugin']}
           when :workflow
             command=self.options.get_next_command([:list, :status, :inputs, :details, :start, :export])
-            unless [:list, :status].include?(command)
-              wf_id=self.options.get_option(:id,:mandatory)
+            unless [:list].include?(command)
+              wf_id=self.instance_identifier()
             end
             case command
             when :status
-              result=call_API('workflows_status')[:data]
+              options={}
+              options[:id]=wf_id unless wf_id.eql?('ALL')
+              result=call_API('workflows_status',options)[:data]
               return {:type=>:object_list,:data=>result['workflows']['workflow']}
             when :list
               result=call_API('workflows_list',id: 0)[:data]