aspera-cli 4.0.0.pre2 → 4.2.0

data/lib/aspera/cli/plugins/console.rb

@@ -14,13 +14,13 @@ module Aspera
           self.options.parse_options!
         end
 
-        ACTIONS=[:transfer,:nagios_check]
+        ACTIONS=[:transfer,:health]
 
         def execute_action
           api_console=basic_auth_api('api')
           command=self.options.get_next_command(ACTIONS)
           case command
-          when :nagios_check
+          when :health
             nagios=Nagios.new
             begin
               api_console.read('ssh_keys')

data/lib/aspera/cli/plugins/faspex.rb

@@ -44,7 +44,7 @@ module Aspera
           result={
             :base_url => "#{publink_uri.scheme}://#{publink_uri.host}#{port_add}#{base}",
             :subpath  => subpath,
-            :query    => URI::decode_www_form(publink_uri.query).inject({}){|m,v|m[v.first]=v.last;m}
+            :query    => URI::decode_www_form(publink_uri.query).inject({}){|h,v|h[v.first]=v.last;h}
           }
           Log.dump('publink',result)
           return result
@@ -52,7 +52,7 @@ module Aspera
 
         # get faspe: URI from entry in xml, and fix problems..
         def self.get_fasp_uri_from_entry(entry)
-          raise CliBadArgument, 'package is empty' unless entry.has_key?('link')
+          raise CliBadArgument, 'package has no link (deleted?)' unless entry.has_key?('link')
           result=entry['link'].select{|e| e['rel'].eql?('package')}.first['href']
           # tags in the end of URL is not well % encoded... there are "=" that should be %3D
           # TODO: enter ticket to Faspex ?
@@ -72,7 +72,7 @@ module Aspera
         def self.get_source_id(source_list,source_name)
           source_ids=source_list.select { |i| i['name'].eql?(source_name) }
           if source_ids.empty?
-            raise CliError,"No such Faspex source #{field_sym.to_s}: #{field_value} in [#{source_list.map{|i| i[field_sym.to_s]}.join(', ')}]"
+            raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
           end
           return source_ids.first['id']
         end
@@ -101,19 +101,26 @@ module Aspera
           return @api_v4
         end
 
-        ACTIONS=[ :nagios_check,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
+        ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
 
         # we match recv command on atom feed on this field
         PACKAGE_MATCH_FIELD='package_id'
 
         def mailbox_all_entries
-          mailbox=self.options.get_option(:box,:mandatory).to_s
+          my_user_name=self.options.get_option(:username,:mandatory)
+          mailbox=self.options.get_option(:box,:mandatory)
           all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
           all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
           Log.dump(:all_inbox_data,all_inbox_data)
           result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
           result.each do |e|
-            e[PACKAGE_MATCH_FIELD]=e['to'].first['recipient_delivery_id'].first
+            case mailbox
+            when :inbox,:archive
+              recipient=e['to'].select{|i|i['name'].first.eql?(my_user_name)}.first
+              e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
+            when :sent
+              e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
+            end
           end
           # remove dropbox packages
           result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
@@ -155,7 +162,7 @@ module Aspera
         def execute_action
           command=self.options.get_next_command(ACTIONS)
           case command
-          when :nagios_check
+          when :health
             nagios=Nagios.new
             begin
               api_v3.read('me')
@@ -241,7 +248,7 @@ module Aspera
                 # TODO: delivery id is the right one if package was receive by group
                 endpoint=case self.options.get_option(:box,:mandatory)
                 when :inbox,:archive;'received'
-                when :sent;'sent'
+                when :sent; 'sent'
                 end
                 entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
                 package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,5 +1,6 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
+require 'securerandom'
 
 module Aspera
   module Cli
@@ -8,74 +9,112 @@ module Aspera
         VAL_ALL='ALL'
         def initialize(env)
           super(env)
-          #self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
-          #self.options.parse_options!
+          options.add_opt_simple(:client_id,'API client identifier in application')
+          options.add_opt_simple(:client_secret,'API client secret in application')
+          options.add_opt_simple(:redirect_uri,'API client redirect URI')
+          options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
+          options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
+          options.set_option(:auth,:jwt)
+          options.parse_options!
         end
-        ACTIONS=[ :node, :package ]
+        ACTIONS=[ :node, :package, :auth_client ]
 
-        # http://apie-next-ui-shell-dev.mybluemix.net/explorer/catalog/aspera/product/ibm-aspera/api/faspex5-api/spec/openapi
+        def set_api
+          faxpex5_api_base_url=options.get_option(:url,:mandatory)
+          faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
+          faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
+          case options.get_option(:auth,:mandatory)
+          when :web
+            @api_v5=Rest.new({
+              :base_url => faxpex5_api_v5_url,
+              :auth     => {
+              :type           => :oauth2,
+              :base_url       => faxpex5_api_auth_url,
+              :grant          => :web,
+              :state          => SecureRandom.uuid,
+              :client_id      => options.get_option(:client_id,:mandatory),
+              :redirect_uri   => options.get_option(:redirect_uri,:mandatory),
+              #:token_field    =>'auth_token',
+              #:path_token     => 'token',
+              #:path_authorize => 'authorize',
+              #:userpass_body  => {name: faxpex5_username,password: faxpex5_password}
+              }})
+          when :boot
+            @api_v5=Rest.new({
+              :base_url => faxpex5_api_v5_url,
+              :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
+            })
+          when :jwt
+            #raise "JWT to be implemented"
+            app_client_id=options.get_option(:client_id,:mandatory)
+            @api_v5=Rest.new({
+              :base_url => faxpex5_api_v5_url,
+              :auth     => {
+              :type                => :oauth2,
+              :base_url            => faxpex5_api_auth_url,
+              :grant               => :jwt,
+              :client_id           => app_client_id,
+              :client_secret       => options.get_option(:client_secret,:mandatory),
+              #:redirect_uri   => options.get_option(:redirect_uri,:mandatory),
+              :jwt_subject         => "client:#{app_client_id}", # TODO Mmmm
+              :jwt_audience        => app_client_id, # TODO Mmmm
+              :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
+              :jwt_is_f5           => true,
+              :jwt_headers         => {typ: 'JWT'}
+              }})
+          end
+        end
+
+        #
         def execute_action
-          # get parameters
-          faxpex5_api_base_url=self.options.get_option(:url,:mandatory)
-          faxpex5_username=self.options.get_option(:username,:mandatory)
-          faxpex5_password=self.options.get_option(:password,:mandatory)
-          faxpex5_api_base_url+='/api/v5'
-          # create object for REST calls to Shares2
-          api_v5=Rest.new({
-            :base_url => faxpex5_api_base_url,
-            :auth     => {
-            :type           => :oauth2,
-            :base_url       => faxpex5_api_base_url,
-            :grant          => :body_data,
-            :token_field    =>'auth_token',
-            :path_token     => 'authenticate',
-            :path_authorize => :unused,
-            :userpass_body  => {name: faxpex5_username,password: faxpex5_password}
-            }})
-          command=self.options.get_next_command(ACTIONS)
+          set_api
+          command=options.get_next_command(ACTIONS)
           case command
+          when :auth_client
+            api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
+            return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
           when :node
-            return self.entity_action(api_v5,'nodes',nil,:id,nil,true)
+            return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
           when :package
-            command=self.options.get_next_command([:list,:show,:send,:receive])
+            command=options.get_next_command([:list,:show,:send,:receive])
             case command
             when :list
-              parameters=self.options.get_option(:value,:optional)
-              return {:type => :object_list, :data=>api_v5.read('packages',parameters)[:data]['packages']}
+              parameters=options.get_option(:value,:optional)
+              return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
             when :show
-              id=self.options.get_option(:id,:mandatory)
-              return {:type => :single_object, :data=>api_v5.read("packages/#{id}")[:data]}
+              id=options.get_option(:id,:mandatory)
+              return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
             when :send
-              parameters=self.options.get_option(:value,:mandatory)
+              parameters=options.get_option(:value,:mandatory)
               raise CliBadArgument,'package value must be hash, refer to API' unless parameters.is_a?(Hash)
-              package=api_v5.create('packages',parameters)[:data]
-              transfer_spec=api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
+              package=@api_v5.create('packages',parameters)[:data]
+              transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
               transfer_spec.delete('authentication')
               return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
             when :receive
               pkg_type='received'
-              pack_id=self.options.get_option(:id,:mandatory)
+              pack_id=options.get_option(:id,:mandatory)
               package_ids=[pack_id]
               skip_ids_data=[]
               skip_ids_persistency=nil
-              if self.options.get_option(:once_only,:mandatory)
+              if options.get_option(:once_only,:mandatory)
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data: skip_ids_data,
-                ids:  ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),pkg_type])
+                ids:  ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
               end
               if pack_id.eql?(VAL_ALL)
                 # todo: if packages have same name, they will overwrite
-                parameters=self.options.get_option(:value,:optional)
+                parameters=options.get_option(:value,:optional)
                 parameters||={"type"=>"received","subtype"=>"mypackages","limit"=>1000}
                 raise CliBadArgument,'value filter must be hash (API GET)' unless parameters.is_a?(Hash)
-                package_ids=api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
+                package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
                 package_ids.select!{|i|!skip_ids_data.include?(i)}
               end
               result_transfer=[]
               package_ids.each do |id|
                 # TODO: allow from sent as well ?
-                transfer_spec=api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
+                transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
                 transfer_spec.delete('authentication')
                 statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
                 result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})

data/lib/aspera/cli/plugins/node.rb

@@ -88,7 +88,7 @@ module Aspera
           raise StandardError,"expect: nil, String or Array"
         end
 
-        SIMPLE_ACTIONS=[:nagios_check,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
+        SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
 
         COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
 
@@ -96,7 +96,7 @@ module Aspera
         # prefix_path is used to list remote sources in Faspex
         def execute_simple_common(command,prefix_path)
           case command
-          when :nagios_check
+          when :health
             nagios=Nagios.new
             begin
               info=@api_node.read('info')[:data]
@@ -341,7 +341,7 @@ module Aspera
               raise "error"
             end
           when :access_key
-            return self.entity_action(@api_node,'access_keys',['id','root_file_id','storage'],:id,'self')
+            return self.entity_action(@api_node,'access_keys',nil,:id,'self')
           when :service
             command=self.options.get_next_command([ :list, :create, :delete])
             if [:delete].include?(command)

data/lib/aspera/cli/plugins/preview.rb

@@ -4,6 +4,7 @@ require 'aspera/preview/options'
 require 'aspera/preview/utils'
 require 'aspera/preview/file_types'
 require 'aspera/persistency_action_once'
+require 'aspera/node'
 require 'aspera/hash_ext'
 require 'date'
 require 'securerandom'
@@ -56,9 +57,11 @@ module Aspera
           self.options.add_opt_simple(:case,'basename of output for for test')
           self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
           self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
+          self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
           self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
           self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
           self.options.set_option(:temp_folder,Dir.tmpdir)
+          self.options.set_option(:mimemagic,:false)
 
           # add other options for generator (and set default values)
           Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
@@ -100,8 +103,6 @@ module Aspera
           return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
         end
 
-        ACTIONS=[:scan,:events,:trevents,:check,:test]
-
         # /files/id/files is normally cached in redis, but we can discard the cache
         # but /files/id is not cached
         def get_folder_entries(file_id,request_args=nil)
@@ -112,14 +113,23 @@ module Aspera
         end
 
         # old version based on folders
-        def process_transfer_events(iteration_token)
+        def process_trevents(iteration_token)
           events_filter={
             'access_key'=>@access_key_self['id'],
             'type'=>'download.ended'
           }
           # optionally by iteration token
           events_filter['iteration_token']=iteration_token unless iteration_token.nil?
-          events=@api_node.read('events',events_filter)[:data]
+          begin
+            events=@api_node.read('events',events_filter)[:data]
+          rescue RestCallError => e
+            if e.message.include?('Invalid iteration_token')
+              Log.log.warn("Retrying without iteration token: #{e}")
+              events_filter.delete('iteration_token')
+              retry
+            end
+            raise e
+          end
           return if events.empty?
           events.each do |event|
             next unless event['data']['direction'].eql?('receive')
@@ -137,7 +147,7 @@ module Aspera
         end
 
         # requests recent events on node api and process newly modified folders
-        def process_file_events(iteration_token)
+        def process_events(iteration_token)
           # get new file creation by access key (TODO: what if file already existed?)
           events_filter={
             'access_key'=>@access_key_self['id'],
@@ -276,7 +286,7 @@ module Aspera
               end
             end
             # need generator for further checks
-            gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'],false)
+            gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
             # get conversion_type (if known) and check if supported
             next false unless gen_info[:generator].supported?
             # shall we skip it ?
@@ -313,12 +323,14 @@ module Aspera
         end # generate_preview
 
         # scan all files in provided folder entry
+        # @param scan_start subpath to start folder scan inside
         def scan_folder_files(top_entry,scan_start=nil)
           if !scan_start.nil?
             # canonical path: start with / and ends with /
             scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
             scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
           end
+          filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
           Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
           # don't use recursive call, use list instead
           entries_to_process=[top_entry]
@@ -334,7 +346,11 @@ module Aspera
             Log.log.debug("item:#{entry}")
             case entry['type']
             when 'file'
-              generate_preview(entry)
+              if filter_block.call(entry)
+                generate_preview(entry)
+              else
+                Log.log.debug('skip by filter')
+              end
             when 'link'
               Log.log.debug('Ignoring link.')
             when 'folder'
@@ -360,6 +376,8 @@ module Aspera
           end
         end
 
+        ACTIONS=[:scan,:events,:trevents,:check,:test]
+
         def execute_action
           command=self.options.get_next_command(ACTIONS)
           unless [:check,:test].include?(command)
@@ -402,6 +420,7 @@ module Aspera
               end
             end
           end
+          Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
           case command
           when :scan
             scan_path=self.options.get_option(:scan_path,:optional)
@@ -417,30 +436,20 @@ module Aspera
             end
             scan_folder_files(folder_info,scan_path)
             return Main.result_status('scan finished')
-          when :events
+          when :events,:trevents
             iteration_data=[]
             iteration_persistency=nil
             if self.options.get_option(:once_only,:mandatory)
               iteration_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
               data: iteration_data,
-              ids:  ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
+              ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
             end
-            iteration_data[0]=process_file_events(iteration_data[0])
-            iteration_persistency.save unless iteration_persistency.nil?
-            return Main.result_status('events finished')
-          when :trevents
-            iteration_data=[]
-            iteration_persistency=nil
-            if self.options.get_option(:once_only,:mandatory)
-              iteration_persistency=PersistencyActionOnce.new(
-              manager: @agents[:persistency],
-              data: iteration_data,
-              ids:  ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
-            end
-            iteration_data[0]=process_transfer_events(iteration_data[0])
+
+            # call method specified
+            iteration_data[0]=send("process_#{command}",iteration_data[0])
             iteration_persistency.save unless iteration_persistency.nil?
-            return Main.result_status('trevents finished')
+            return Main.result_status("#{command} finished")
           when :check
             Aspera::Preview::Utils.check_tools(@skip_types)
             return Main.result_status('tools validated')
@@ -448,8 +457,9 @@ module Aspera
             format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
             source = self.options.get_next_argument('source file')
             dest=preview_filename(format,self.options.get_option(:case,:optional))
-            g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
-            raise "format not supported: #{format}" unless g.supported?
+            g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
+            raise "cannot find file type for #{source}" if g.conversion_type.nil?
+            raise "out format #{format} not supported" unless g.supported?
             g.generate
             return Main.result_status("generated: #{dest}")
           else

data/lib/aspera/cli/plugins/server.rb

@@ -62,7 +62,7 @@ module Aspera
           end.select{|i|!i.nil?}
         end
 
-        ACTIONS=[:nagios,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
+        ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
 
         def execute_action
           server_uri=URI.parse(self.options.get_option(:url,:mandatory))
@@ -93,12 +93,14 @@ module Aspera
               ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
               ssh_keys.map!{|p|File.expand_path(p)}
               Log.log.debug("ssh keys=#{ssh_keys}")
-              ssh_options[:keys]=ssh_keys
-              server_transfer_spec['EX_ssh_key_paths']=ssh_keys
-              ssh_keys.each do |k|
-                Log.log.warn("no such key file: #{k}") unless File.exist?(k)
+              if !ssh_keys.empty?
+                ssh_options[:keys]=ssh_keys
+                server_transfer_spec['EX_ssh_key_paths']=ssh_keys
+                ssh_keys.each do |k|
+                  Log.log.warn("no such key file: #{k}") unless File.exist?(k)
+                end
+                cred_set=true
               end
-              cred_set=true
             end
             raise 'either password or key must be provided' if !cred_set
             shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
@@ -114,9 +116,9 @@ module Aspera
           command=:rm if command.eql?(:delete)
           command=:mv if command.eql?(:rename)
           case command
-          when :nagios
+          when :health
             nagios=Nagios.new
-            command_nagios=self.options.get_next_command([ :app_services, :transfer ])
+            command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
             case command_nagios
             when :app_services
               # will not work with aspshell, requires Linux/bash
@@ -145,6 +147,19 @@ module Aspera
               else
                 nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
               end
+            when :asctlstatus
+              realcmd='asctl'
+              prefix=self.options.get_option(:cmd_prefix,:optional)
+              realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
+              result=shell_executor.execute(realcmd.split(' '))
+              data=asctl_parse(result)
+              data.each do |i|
+                if i['state'].eql?('running')
+                  nagios.add_ok(i['process'],i['state'])
+                else
+                  nagios.add_critical(i['process'],i['state'])
+                end
+              end
             else raise "ERROR"
             end
             return nagios.result