aspera-cli 4.1.0 → 4.3.0

data/lib/aspera/cli/plugins/faspex.rb

@@ -7,8 +7,10 @@ require 'aspera/persistency_action_once'
 require 'aspera/open_application'
 require 'aspera/fasp/uri'
 require 'aspera/nagios'
+require 'aspera/id_generator'
 require 'xmlsimple'
 require 'json'
+require 'cgi'
 
 module Aspera
   module Cli
@@ -17,7 +19,17 @@ module Aspera
         KEY_NODE='node'
         KEY_PATH='path'
         VAL_ALL='ALL'
-        private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL
+        # added field in result that identifies the package
+        PACKAGE_MATCH_FIELD='package_id'
+        # list of supported atoms
+        ATOM_MAILBOXES=[:inbox, :archive, :sent]
+        # number is added by CLI
+        MAX_ITEMS='max'
+        MAX_PAGES='pmax'
+        ATOM_PARAMS=['page', 'count', 'startIndex', MAX_ITEMS, MAX_PAGES]
+        PUB_LINK_EXTERNAL_MATCH='external_deliveries/'
+        private_constant :KEY_NODE,:KEY_PATH,:VAL_ALL,:PACKAGE_MATCH_FIELD,:ATOM_MAILBOXES,:PUB_LINK_EXTERNAL_MATCH
+
         def initialize(env)
           @api_v3=nil
           @api_v4=nil
@@ -26,7 +38,8 @@ module Aspera
           self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
           self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
           self.options.add_opt_simple(:storage,'Faspex local storage definition')
-          self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
+          self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
+          self.options.add_opt_list(:box,ATOM_MAILBOXES,'package box')
           self.options.set_option(:box,:inbox)
           self.options.parse_options!
         end
@@ -72,7 +85,7 @@ module Aspera
         def self.get_source_id(source_list,source_name)
           source_ids=source_list.select { |i| i['name'].eql?(source_name) }
           if source_ids.empty?
-            raise CliError,"No such Faspex source #{field_sym.to_s}: #{field_value} in [#{source_list.map{|i| i[field_sym.to_s]}.join(', ')}]"
+            raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
           end
           return source_ids.first['id']
         end
@@ -101,29 +114,57 @@ module Aspera
           return @api_v4
         end
 
-        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
-
-        # we match recv command on atom feed on this field
-        PACKAGE_MATCH_FIELD='package_id'
-
+        # query supports : {"startIndex":10,"count":1,"page":109}
         def mailbox_all_entries
-          my_user_name=self.options.get_option(:username,:mandatory)
+          recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
+          # mailbox is in ATOM_MAILBOXES
           mailbox=self.options.get_option(:box,:mandatory)
-          all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
-          all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
-          Log.dump(:all_inbox_data,all_inbox_data)
-          result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
-          result.each do |e|
-            case mailbox
-            when :inbox,:archive
-              recipient=e['to'].select{|i|i['name'].first.eql?(my_user_name)}.first
-              e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
-            when :sent
-              e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
+          # parameters
+          mailbox_query=self.options.get_option(:query,:optional)
+          max_items=nil
+          max_pages=nil
+          result=[]
+          if !mailbox_query.nil?
+            raise "query: must be Hash or nil" unless mailbox_query.is_a?(Hash)
+            raise "query: supported params: #{ATOM_PARAMS}" unless (mailbox_query.keys-ATOM_PARAMS).empty?
+            raise "query: startIndex and page are exclusive" if mailbox_query.has_key?('startIndex') and mailbox_query.has_key?('page')
+            max_items=mailbox_query[MAX_ITEMS]
+            mailbox_query.delete(MAX_ITEMS)
+            max_pages=mailbox_query[MAX_PAGES]
+            mailbox_query.delete(MAX_PAGES)
+          end
+          loop do
+            atom_xml=api_v3.call({operation: 'GET',subpath: "#{mailbox}.atom",headers: {'Accept'=>'application/xml'},url_params: mailbox_query})[:http].body
+            box_data=XmlSimple.xml_in(atom_xml, {'ForceArray' => true})
+            Log.dump(:box_data,box_data)
+            items=box_data.has_key?('entry') ? box_data['entry'] : []
+            Log.log.debug("new items: #{items.count}")
+            # it is the end if page is empty
+            break if items.empty?
+            items.each do |package|
+              package[PACKAGE_MATCH_FIELD]=case mailbox
+              when :inbox,:archive
+                recipient=package['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
+                recipient.nil? ? nil : recipient['recipient_delivery_id'].first
+              else # :sent
+                package['delivery_id'].first
+              end
+              # keep only those for the specified recipient
+              result.push(package) unless package[PACKAGE_MATCH_FIELD].nil?
             end
+            Log.log.debug("total items: #{result.count}")
+            # reach the limit ?
+            break if !max_items.nil? and result.count > max_items
+            link=box_data['link'].select{|i|i['rel'].eql?('next')}.first
+            Log.log.debug("link: #{link}")
+            # no next link
+            break if link.nil?
+            # replace parameters with the ones from next link
+            params=CGI.parse(URI.parse(link['href']).query)
+            mailbox_query=params.keys.inject({}){|m,i|;m[i]=params[i].first;m}
+            Log.log.debug("query: #{mailbox_query}")
+            break if !max_pages.nil? and mailbox_query['page'].to_i > max_pages
           end
-          # remove dropbox packages
-          result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
           return result
         end
 
@@ -159,6 +200,8 @@ module Aspera
           return pkgdatares.first
         end
 
+        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
+
         def execute_action
           command=self.options.get_next_command(ACTIONS)
           case command
@@ -208,52 +251,59 @@ module Aspera
               #Log.dump('transfer_spec',transfer_spec)
               return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
             when :recv
-              public_link_url=self.options.get_option(:link,:optional)
-              if !public_link_url.nil?
-                link_data=self.class.get_link_data(public_link_url)
-                if !link_data[:subpath].match(%r{external_deliveries/})
-                  raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
+              link_url=self.options.get_option(:link,:optional)
+              # list of faspex ID/URI to download
+              pkg_id_uri=nil
+              skip_ids_data=[]
+              skip_ids_persistency=nil
+              case link_url
+              when nil # usual case: no link
+                if self.options.get_option(:once_only,:mandatory)
+                  skip_ids_persistency=PersistencyActionOnce.new(
+                  manager: @agents[:persistency],
+                  data:    skip_ids_data,
+                  id:      IdGenerator.from_list(['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s]))
+                end
+                # get command line parameters
+                delivid=self.options.get_option(:id,:mandatory)
+                if delivid.eql?(VAL_ALL)
+                  pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
+                  # TODO : remove ids from skip not present in inbox
+                  # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
+                  pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
+                else
+                  recipient=options.get_option(:recipient,:optional)
+                  if !recipient.nil? and recipient.start_with?('*')
+                    raise "Dropbox and Workgroup packages should use link option with faspe:"
+                  end
+                  # TODO: delivery id is the right one if package was receive by workgroup
+                  endpoint=case self.options.get_option(:box,:mandatory)
+                  when :inbox,:archive;'received'
+                  when :sent; 'sent'
+                  end
+                  entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
+                  package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
+                  pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
+                end
+              when /^faspe:/
+                pkg_id_uri=[{:id=>'package',:uri=>link_url}]
+              else
+                link_data=self.class.get_link_data(link_url)
+                if !link_data[:subpath].start_with?(PUB_LINK_EXTERNAL_MATCH)
+                  raise CliBadArgument,"Pub link is #{link_data[:subpath]}. Expecting #{PUB_LINK_EXTERNAL_MATCH}"
                 end
-                # Note: unauthenticated API
+                # Note: unauthenticated API (autorization is in url params)
                 api_public_link=Rest.new({:base_url=>link_data[:base_url]})
                 pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
                 if !pkgdatares[:http].body.start_with?('<?xml ')
-                  OpenApplication.instance.uri(public_link_url)
+                  OpenApplication.instance.uri(link_url)
                   raise CliError, 'no such package'
                 end
                 package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
+                Log.dump(:package_entry,package_entry)
                 transfer_uri=self.class.get_fasp_uri_from_entry(package_entry)
-                transfer_spec=Fasp::Uri.new(transfer_uri).transfer_spec
-                transfer_spec['direction']='receive'
-                return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
+                pkg_id_uri=[{:id=>package_entry['id'],:uri=>transfer_uri}]
               end # public link
-              # get command line parameters
-              delivid=self.options.get_option(:id,:mandatory)
-              # list of faspex ID/URI to download
-              pkg_id_uri=nil
-              skip_ids_data=[]
-              skip_ids_persistency=nil
-              if self.options.get_option(:once_only,:mandatory)
-                skip_ids_persistency=PersistencyActionOnce.new(
-                manager: @agents[:persistency],
-                data: skip_ids_data,
-                ids:  ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
-              end
-              if delivid.eql?(VAL_ALL)
-                pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
-                # TODO : remove ids from skip not present in inbox
-                # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
-                pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
-              else
-                # TODO: delivery id is the right one if package was receive by group
-                endpoint=case self.options.get_option(:box,:mandatory)
-                when :inbox,:archive;'received'
-                when :sent; 'sent'
-                end
-                entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
-                package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
-                pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
-              end
               Log.dump(:pkg_id_uri,pkg_id_uri)
               return Main.result_status('no package') if pkg_id_uri.empty?
               result_transfer=[]
@@ -262,20 +312,17 @@ module Aspera
                 # NOTE: only external users have token in faspe: link !
                 if !transfer_spec.has_key?('token')
                   sanitized=id_uri[:uri].gsub('&','&amp;')
-                  # TODO: file jira
-                  #XXsanitized.gsub!(/%3D%3D$/,'==')
-                  #XXsanitized.gsub!(/%3D$/,'=')
                   xmlpayload='<?xml version="1.0" encoding="UTF-8"?><url-list xmlns="http://schemas.asperasoft.com/xml/url-list"><url href="'+sanitized+'"/></url-list>'
                   transfer_spec['token']=api_v3.call({:operation=>'POST',:subpath=>'issue-token?direction=down',:headers=>{'Accept'=>'text/plain','Content-Type'=>'application/vnd.aspera.url-list+xml'},:text_body_params=>xmlpayload})[:http].body
                 end
                 transfer_spec['direction']='receive'
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
-                result_transfer.push({'package'=>id_uri[:id],'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>id_uri[:id],Main::STATUS_FIELD=>statuses})
                 # skip only if all sessions completed
                 skip_ids_data.push(id_uri[:id]) if TransferAgent.session_status(statuses).eql?(:success)
               end
               skip_ids_persistency.save unless skip_ids_persistency.nil?
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             end
           when :source
             command_source=self.options.get_next_command([ :list, :id, :name ])

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,5 +1,6 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'securerandom'
 
 module Aspera
@@ -17,14 +18,20 @@ module Aspera
           options.set_option(:auth,:jwt)
           options.parse_options!
         end
-        ACTIONS=[ :node, :package, :auth_client ]
 
         def set_api
-          faxpex5_api_base_url=options.get_option(:url,:mandatory)
-          faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
-          faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
+          @faxpex5_api_base_url=options.get_option(:url,:mandatory)
+          faxpex5_api_v5_url="#{@faxpex5_api_base_url}/api/v5"
+          faxpex5_api_auth_url="#{@faxpex5_api_base_url}/auth"
           case options.get_option(:auth,:mandatory)
+          when :boot
+            # the password here is the token copied directly from browser in developer mode
+            @api_v5=Rest.new({
+              :base_url => faxpex5_api_v5_url,
+              :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
+            })
           when :web
+            # opens a browser and ask user to auth using web
             @api_v5=Rest.new({
               :base_url => faxpex5_api_v5_url,
               :auth     => {
@@ -34,64 +41,43 @@ module Aspera
               :state          => SecureRandom.uuid,
               :client_id      => options.get_option(:client_id,:mandatory),
               :redirect_uri   => options.get_option(:redirect_uri,:mandatory),
-              #:token_field    =>'auth_token',
-              #:path_token     => 'token',
-              #:path_authorize => 'authorize',
-              #:userpass_body  => {name: faxpex5_username,password: faxpex5_password}
               }})
-          when :boot
-            @api_v5=Rest.new({
-              :base_url => faxpex5_api_v5_url,
-              :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
-            })
           when :jwt
-            #raise "JWT to be implemented"
+            # currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
+            app_client_id=options.get_option(:client_id,:mandatory)
             @api_v5=Rest.new({
-              :base_url => faxpex5_api_base_url,
+              :base_url => faxpex5_api_v5_url,
               :auth     => {
               :type                => :oauth2,
               :base_url            => faxpex5_api_auth_url,
               :grant               => :jwt,
-              :client_id           => options.get_option(:client_id,:mandatory),
+              :f5_username         => options.get_option(:username,:mandatory),
+              :f5_password         => options.get_option(:password,:mandatory),
+              :client_id           => app_client_id,
               :client_secret       => options.get_option(:client_secret,:mandatory),
-              #:redirect_uri   => options.get_option(:redirect_uri,:mandatory),
-              :jwt_subject         => "client:#{options.get_option(:client_id,:mandatory)}", # TODO Mmmm
+              :jwt_subject         => "client:#{app_client_id}", # TODO Mmmm
+              :jwt_audience        => app_client_id, # TODO Mmmm
               :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
-              :jwt_audience =>options.get_option(:client_id,:mandatory), # TODO Mmmm
-              #:token_field    =>'auth_token',
-              #:path_token     => 'authenticate',
-              #:path_authorize => :unused,
-              #:userpass_body  => {name: options.get_option(:username,:mandatory),password: options.get_option(:password,:mandatory)}
+              :jwt_headers         => {typ: 'JWT'}
               }})
-            #  former version
-            #            # get parameters
-            #            faxpex5_username=options.get_option(:username,:mandatory)
-            #            faxpex5_password=options.get_option(:password,:mandatory)
-            #            # create object for REST calls to Shares2
-            #            @api_v5=Rest.new({
-            #              :base_url => faxpex5_api_base_url,
-            #              :auth     => {
-            #              :type           => :oauth2,
-            #              :base_url       => faxpex5_api_base_url,
-            #              :grant          => :body_data,
-            #              :token_field    =>'auth_token',
-            #              :path_token     => 'authenticate',
-            #              :path_authorize => :unused,
-            #              :userpass_body  => {name: faxpex5_username,password: faxpex5_password}
-            #              }})
           end
         end
 
+        ACTIONS=[ :node, :package, :auth_client, :jobs ]
+
         #
         def execute_action
           set_api
           command=options.get_next_command(ACTIONS)
           case command
           when :auth_client
-            api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
+            api_auth=Rest.new(@api_v5.params.merge({base_url: "#{@faxpex5_api_base_url}/auth"}))
             return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
           when :node
             return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
+          when :jobs
+            # to test JWT
+            return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
           when :package
             command=options.get_next_command([:list,:show,:send,:receive])
             case command
@@ -103,7 +89,7 @@ module Aspera
               return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
             when :send
               parameters=options.get_option(:value,:mandatory)
-              raise CliBadArgument,'package value must be hash, refer to API' unless parameters.is_a?(Hash)
+              raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
               package=@api_v5.create('packages',parameters)[:data]
               transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
               transfer_spec.delete('authentication')
@@ -115,16 +101,17 @@ module Aspera
               skip_ids_data=[]
               skip_ids_persistency=nil
               if options.get_option(:once_only,:mandatory)
+                # read ids from persistency
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
-                data: skip_ids_data,
-                ids:  ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
+                data:    skip_ids_data,
+                id:      IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
               end
               if pack_id.eql?(VAL_ALL)
-                # todo: if packages have same name, they will overwrite
+                # TODO: if packages have same name, they will overwrite
                 parameters=options.get_option(:value,:optional)
-                parameters||={"type"=>"received","subtype"=>"mypackages","limit"=>1000}
-                raise CliBadArgument,'value filter must be hash (API GET)' unless parameters.is_a?(Hash)
+                parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
+                raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
                 package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
                 package_ids.select!{|i|!skip_ids_data.include?(i)}
               end
@@ -134,12 +121,12 @@ module Aspera
                 transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
                 transfer_spec.delete('authentication')
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
-                result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>id,Main::STATUS_FIELD=>statuses})
                 # skip only if all sessions completed
                 skip_ids_data.push(id) if TransferAgent.session_status(statuses).eql?(:success)
               end
               skip_ids_persistency.save unless skip_ids_persistency.nil?
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             end
           end
         end

data/lib/aspera/cli/plugins/node.rb

@@ -1,6 +1,7 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/nagios'
 require 'aspera/hash_ext'
+require 'aspera/id_generator'
 require 'base64'
 require 'zlib'
 
@@ -271,8 +272,8 @@ module Aspera
             if self.options.get_option(:once_only,:mandatory)
               skip_ids_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
-              data: iteration_data,
-              ids:  ['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid])
+              data:    iteration_data,
+              id:      IdGenerator.from_list(['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid]))
               unless iteration_data.first.nil?
                 data.select!{|l| l['fnid'].to_i>iteration_data.first}
               end