aspera-cli 4.0.0 → 4.1.0

data/lib/aspera/cli/plugins/node.rb

@@ -88,7 +88,7 @@ module Aspera
           raise StandardError,"expect: nil, String or Array"
         end
 
-        SIMPLE_ACTIONS=[:nagios_check,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
+        SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
 
         COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
 
@@ -96,7 +96,7 @@ module Aspera
         # prefix_path is used to list remote sources in Faspex
         def execute_simple_common(command,prefix_path)
           case command
-          when :nagios_check
+          when :health
             nagios=Nagios.new
             begin
               info=@api_node.read('info')[:data]
@@ -341,7 +341,7 @@ module Aspera
               raise "error"
             end
           when :access_key
-            return self.entity_action(@api_node,'access_keys',['id','root_file_id','storage'],:id,'self')
+            return self.entity_action(@api_node,'access_keys',nil,:id,'self')
           when :service
             command=self.options.get_next_command([ :list, :create, :delete])
             if [:delete].include?(command)

data/lib/aspera/cli/plugins/preview.rb

@@ -56,9 +56,11 @@ module Aspera
           self.options.add_opt_simple(:case,'basename of output for for test')
           self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
           self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
+          self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
           self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
           self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
           self.options.set_option(:temp_folder,Dir.tmpdir)
+          self.options.set_option(:mimemagic,:false)
 
           # add other options for generator (and set default values)
           Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
@@ -100,8 +102,6 @@ module Aspera
           return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
         end
 
-        ACTIONS=[:scan,:events,:trevents,:check,:test]
-
         # /files/id/files is normally cached in redis, but we can discard the cache
         # but /files/id is not cached
         def get_folder_entries(file_id,request_args=nil)
@@ -112,14 +112,23 @@ module Aspera
         end
 
         # old version based on folders
-        def process_transfer_events(iteration_token)
+        def process_trevents(iteration_token)
           events_filter={
             'access_key'=>@access_key_self['id'],
             'type'=>'download.ended'
           }
           # optionally by iteration token
           events_filter['iteration_token']=iteration_token unless iteration_token.nil?
-          events=@api_node.read('events',events_filter)[:data]
+          begin
+            events=@api_node.read('events',events_filter)[:data]
+          rescue RestCallError => e
+            if e.message.include?('Invalid iteration_token')
+              Log.log.warn("Retrying without iteration token: #{e}")
+              events_filter.delete('iteration_token')
+              retry
+            end
+            raise e
+          end
           return if events.empty?
           events.each do |event|
             next unless event['data']['direction'].eql?('receive')
@@ -137,7 +146,7 @@ module Aspera
         end
 
         # requests recent events on node api and process newly modified folders
-        def process_file_events(iteration_token)
+        def process_events(iteration_token)
           # get new file creation by access key (TODO: what if file already existed?)
           events_filter={
             'access_key'=>@access_key_self['id'],
@@ -276,7 +285,7 @@ module Aspera
               end
             end
             # need generator for further checks
-            gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'],false)
+            gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
             # get conversion_type (if known) and check if supported
             next false unless gen_info[:generator].supported?
             # shall we skip it ?
@@ -360,6 +369,8 @@ module Aspera
           end
         end
 
+        ACTIONS=[:scan,:events,:trevents,:check,:test]
+
         def execute_action
           command=self.options.get_next_command(ACTIONS)
           unless [:check,:test].include?(command)
@@ -402,6 +413,7 @@ module Aspera
               end
             end
           end
+          Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
           case command
           when :scan
             scan_path=self.options.get_option(:scan_path,:optional)
@@ -417,30 +429,18 @@ module Aspera
             end
             scan_folder_files(folder_info,scan_path)
             return Main.result_status('scan finished')
-          when :events
-            iteration_data=[]
-            iteration_persistency=nil
-            if self.options.get_option(:once_only,:mandatory)
-              iteration_persistency=PersistencyActionOnce.new(
-              manager: @agents[:persistency],
-              data: iteration_data,
-              ids:  ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
-            end
-            iteration_data[0]=process_file_events(iteration_data[0])
-            iteration_persistency.save unless iteration_persistency.nil?
-            return Main.result_status('events finished')
-          when :trevents
+          when :events,:trevents
             iteration_data=[]
             iteration_persistency=nil
             if self.options.get_option(:once_only,:mandatory)
               iteration_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
               data: iteration_data,
-              ids:  ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
+              ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
             end
-            iteration_data[0]=process_transfer_events(iteration_data[0])
+            iteration_data[0]=send("process_#{command}",iteration_data[0])
             iteration_persistency.save unless iteration_persistency.nil?
-            return Main.result_status('trevents finished')
+            return Main.result_status("#{command} finished")
           when :check
             Aspera::Preview::Utils.check_tools(@skip_types)
             return Main.result_status('tools validated')
@@ -448,8 +448,9 @@ module Aspera
             format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
             source = self.options.get_next_argument('source file')
             dest=preview_filename(format,self.options.get_option(:case,:optional))
-            g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
-            raise "format not supported: #{format}" unless g.supported?
+            g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
+            raise "cannot find file type for #{source}" if g.conversion_type.nil?
+            raise "out format #{format} not supported" unless g.supported?
             g.generate
             return Main.result_status("generated: #{dest}")
           else

data/lib/aspera/cli/plugins/server.rb

@@ -62,7 +62,7 @@ module Aspera
           end.select{|i|!i.nil?}
         end
 
-        ACTIONS=[:nagios,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
+        ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
 
         def execute_action
           server_uri=URI.parse(self.options.get_option(:url,:mandatory))
@@ -93,12 +93,14 @@ module Aspera
               ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
               ssh_keys.map!{|p|File.expand_path(p)}
               Log.log.debug("ssh keys=#{ssh_keys}")
-              ssh_options[:keys]=ssh_keys
-              server_transfer_spec['EX_ssh_key_paths']=ssh_keys
-              ssh_keys.each do |k|
-                Log.log.warn("no such key file: #{k}") unless File.exist?(k)
+              if !ssh_keys.empty?
+                ssh_options[:keys]=ssh_keys
+                server_transfer_spec['EX_ssh_key_paths']=ssh_keys
+                ssh_keys.each do |k|
+                  Log.log.warn("no such key file: #{k}") unless File.exist?(k)
+                end
+                cred_set=true
               end
-              cred_set=true
             end
             raise 'either password or key must be provided' if !cred_set
             shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
@@ -114,9 +116,9 @@ module Aspera
           command=:rm if command.eql?(:delete)
           command=:mv if command.eql?(:rename)
           case command
-          when :nagios
+          when :health
             nagios=Nagios.new
-            command_nagios=self.options.get_next_command([ :app_services, :transfer ])
+            command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
             case command_nagios
             when :app_services
               # will not work with aspshell, requires Linux/bash
@@ -145,6 +147,19 @@ module Aspera
               else
                 nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
               end
+            when :asctlstatus
+              realcmd='asctl'
+              prefix=self.options.get_option(:cmd_prefix,:optional)
+              realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
+              result=shell_executor.execute(realcmd.split(' '))
+              data=asctl_parse(result)
+              data.each do |i|
+                if i['state'].eql?('running')
+                  nagios.add_ok(i['process'],i['state'])
+                else
+                  nagios.add_critical(i['process'],i['state'])
+                end
+              end
             else raise "ERROR"
             end
             return nagios.result

data/lib/aspera/cli/transfer_agent.rb

@@ -22,7 +22,7 @@ module Aspera
         @opt_mgr=cli_objects[:options]
         @config=cli_objects[:config]
         # transfer spec overrides provided on command line
-        @transfer_spec_cmdline={}
+        @transfer_spec_cmdline={"create_dir"=>true}
         # the currently selected transfer agent
         @agent=nil
         @progress_listener=Listener::ProgressMulti.new

data/lib/aspera/cli/version.rb

@@ -1,5 +1,5 @@
 module Aspera
   module Cli
-    VERSION = "4.0.0"
+    VERSION = "4.1.0"
   end
 end

data/lib/aspera/fasp/connect.rb

@@ -12,7 +12,9 @@ module Aspera
       private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
       def initialize
         super
-        @connect_app_id=SecureRandom.uuid
+        @connect_settings={
+          'app_id' => SecureRandom.uuid
+        }
         # TODO: start here and create monitor
       end
 
@@ -32,46 +34,51 @@ module Aspera
             OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
             raise StandardError,'Connect is not installed'
           end
-          sleep SLEEP_SEC_BETWEEN_RETRY
+          sleep(SLEEP_SEC_BETWEEN_RETRY)
           retry
         end
         if transfer_spec['direction'] == 'send'
           Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
           transfer_spec.delete('paths')
-          resdata=@connect_api.create('windows/select-open-file-dialog/',{'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>'','aspera_connect_settings'=>{'app_id'=>@connect_app_id}})[:data]
+          resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
           transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
         end
         @request_id=SecureRandom.uuid
         # if there is a token, we ask connect client to use well known ssh private keys
         # instead of asking password
         transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
+        connect_settings=
         connect_transfer_args={
-          'transfer_specs'=>[{
-          'transfer_spec'=>transfer_spec,
-          'aspera_connect_settings'=>{
-          'allow_dialogs'=>true,
-          'app_id'=>@connect_app_id,
-          'request_id'=>@request_id
-          }}]}
+          'aspera_connect_settings'=>@connect_settings.merge({
+          'request_id'               =>@request_id,
+          'allow_dialogs'            =>true,
+          }),
+          'transfer_specs'         =>[{
+          'transfer_spec'            =>transfer_spec,
+          }]}
         # asynchronous anyway
-        @connect_api.create('transfers/start',connect_transfer_args)
+        res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
+        @xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
       end
 
       def wait_for_transfers_completion
-        connect_activity_args={'aspera_connect_settings'=>{'app_id'=>@connect_app_id}}
+        connect_activity_args={'aspera_connect_settings'=>@connect_settings}
         started=false
         spinner=nil
         loop do
-          result=@connect_api.create('transfers/activity',connect_activity_args)[:data]
-          if result['transfers']
-            trdata=result['transfers'].select{|i| i['aspera_connect_settings'] and i['aspera_connect_settings']['request_id'].eql?(@request_id)}.first
-            raise 'problem with connect, please kill it' unless trdata
+          tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
+          if tr_info['transfer_info'].is_a?(Hash)
+            trdata=tr_info['transfer_info']
+            if trdata.nil?
+              Log.log.warn("no session in Connect")
+              break
+            end
             # TODO: get session id
             case trdata['status']
             when 'completed'
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@connect_app_id,'Type'=>'DONE'})
+              notify_end(@connect_settings['app_id'])
               break
-            when 'initiating'
+            when 'initiating','queued'
               if spinner.nil?
                 spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
                 spinner.start
@@ -81,13 +88,13 @@ module Aspera
             when 'running'
               #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
               if !started and trdata['bytes_expected'] != 0
-                notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@connect_app_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['bytes_expected']})
+                notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
                 started=true
               else
-                notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@connect_app_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_written']})
+                notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
               end
             else
-              raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
+              raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
             end
           end
           sleep 1

data/lib/aspera/fasp/http_gw.rb

@@ -2,12 +2,147 @@
 require 'aspera/fasp/manager'
 require 'aspera/log'
 require 'aspera/rest'
+require 'websocket-client-simple'
+require 'securerandom'
+require 'openssl'
+require 'base64'
+require 'json'
+require 'uri'
 
 # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
 module Aspera
   module Fasp
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class HttpGW < Manager
+      # message returned by HTTP GW in case of success
+      OK_MESSAGE='end upload'
+      # refresh rate for progress
+      UPLOAD_REFRESH_SEC=0.5
+      private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
+      # send message on http gw web socket
+      def ws_send(ws,type,data)
+        ws.send(JSON.generate({type => data}))
+      end
+
+      def upload(transfer_spec)
+        # precalculate size
+        total_size=0
+        # currently, files are sent flat
+        source_path=[]
+        transfer_spec['paths'].each do |item|
+          filepath=item['source']
+          item['source']=item['destination']=File.basename(filepath)
+          total_size+=item['file_size']=File.size(filepath)
+          source_path.push(filepath)
+        end
+        session_id=SecureRandom.uuid
+        ws=::WebSocket::Client::Simple::Client.new
+        error=nil
+        received=0
+        ws.on :message do |msg|
+          Log.log.info("ws: message: #{msg.data}")
+          message=msg.data
+          if message.eql?(OK_MESSAGE)
+            received+=1
+          else
+            message.chomp!
+            if message[0].eql?('"') and message[-1].eql?('"')
+              error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
+            else
+              error="expecting quotes in [#{message}]"
+            end
+          end
+        end
+        ws.on :error do |e|
+          error=e
+        end
+        ws.on :open do
+          Log.log.info("ws: open")
+        end
+        ws.on :close do
+          Log.log.info("ws: close")
+        end
+        # open web socket to end point
+        ws.connect("#{@gw_api.params[:base_url]}/upload")
+        while !ws.open? and error.nil? do
+          Log.log.info("ws: wait")
+          sleep(0.2)
+        end
+        notify_begin(session_id,total_size)
+        ws_send(ws,:transfer_spec,transfer_spec)
+        # current file index
+        filenum=0
+        # aggregate size sent
+        sent_bytes=0
+        # last progress event
+        lastevent=Time.now-1
+        transfer_spec['paths'].each do |item|
+          # TODO: on destination write same path?
+          destination_path=item['source']
+          # TODO: get mime type?
+          file_mime_type=''
+          total=item['file_size']
+          # compute total number of slices
+          numslices=1+(total-1)/@upload_chunksize
+          # current slice index
+          slicenum=0
+          File.open(source_path[filenum]) do |file|
+            while !file.eof? do
+              data=file.read(@upload_chunksize)
+              slice_data={
+                name: destination_path,
+                type: file_mime_type,
+                size: total,
+                data: Base64.strict_encode64(data),
+                slice: slicenum,
+                total_slices: numslices,
+                fileIndex: filenum
+              }
+              ws_send(ws,:slice_upload, slice_data)
+              sent_bytes+=data.length
+              currenttime=Time.now
+              if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
+                notify_progress(session_id,sent_bytes)
+                lastevent=currenttime
+              end
+              slicenum+=1
+              raise error unless error.nil?
+            end
+          end
+          filenum+=1
+        end
+        ws.close
+        notify_end(session_id)
+      end
+
+      def download(transfer_spec)
+        transfer_spec['zip_required']||=false
+        transfer_spec['source_root']||='/'
+        # is normally provided by application, like package name
+        if !transfer_spec.has_key?('download_name')
+          # by default it is the name of first file
+          dname=File.basename(transfer_spec['paths'].first['source'])
+          # we remove extension
+          dname=dname.gsub(/\.@gw_api.*$/,'')
+          # ands add indication of number of files if there is more than one
+          if transfer_spec['paths'].length > 1
+            dname=dname+" #{transfer_spec['paths'].length} Files"
+          end
+          transfer_spec['download_name']=dname
+        end
+        creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
+        transfer_uuid=creation['url'].split('/').last
+        if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
+          # it is a zip file if zip is required or there is more than 1 file
+          file_dest=transfer_spec['download_name']+'.zip'
+        else
+          # it is a plain file if we don't require zip and there is only one file
+          file_dest=File.basename(transfer_spec['paths'].first['source'])
+        end
+        file_dest=File.join(transfer_spec['destination_root'],file_dest)
+        @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
+      end
+
       # start FASP transfer based on transfer spec (hash table)
       # note that it is asynchronous
       # HTTP download only supports file list
@@ -15,37 +150,13 @@ module Aspera
         raise "GW URL must be set" unless !@gw_api.nil?
         raise "option: must be hash (or nil)" unless options.is_a?(Hash)
         raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
+        raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        transfer_spec['authentication']||='token'
         case transfer_spec['direction']
         when 'send'
-          # this is a websocket
-          raise "error, not implemented"
+          upload(transfer_spec)
         when 'receive'
-          transfer_spec['zip_required']||=false
-          transfer_spec['authentication']||='token'
-          transfer_spec['source_root']||='/'
-          # is normally provided by application, like package name
-          if !transfer_spec.has_key?('download_name')
-            # by default it is the name of first file
-            dname=File.basename(transfer_spec['paths'].first['source'])
-            # we remove extension
-            dname=dname.gsub(/\.@gw_api.*$/,'')
-            # ands add indication of number of files if there is more than one
-            if transfer_spec['paths'].length > 1
-              dname=dname+" #{transfer_spec['paths'].length} Files"
-            end
-            transfer_spec['download_name']=dname
-          end
-          creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
-          transfer_uuid=creation['url'].split('/').last
-          if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
-            # it is a zip file if zip is required or there is more than 1 file
-            file_dest=transfer_spec['download_name']+'.zip'
-          else
-            # it is a plain file if we don't require zip and there is only one file
-            file_dest=File.basename(transfer_spec['paths'].first['source'])
-          end
-          file_dest=File.join(transfer_spec['destination_root'],file_dest)
-          @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
+          download(transfer_spec)
         else
           raise "error"
         end
@@ -74,6 +185,7 @@ module Aspera
         @gw_api=Rest.new({:base_url => params[:url]})
         api_info = @gw_api.read('info')[:data]
         Log.log.info("#{api_info}")
+        @upload_chunksize=128000 # TODO: configurable ?
       end
 
     end # LocalHttp