aspera-cli 4.0.0.pre3 → 4.2.1

data/lib/aspera/fasp/installation.rb

@@ -22,11 +22,18 @@ module Aspera
       PRODUCT_CLI_V1='Aspera CLI'
       PRODUCT_DRIVE='Aspera Drive'
       PRODUCT_ENTSRV='Enterprise Server'
-      # currently used ascp executable
+      MAX_REDIRECT_SDK=2
+      private_constant :MAX_REDIRECT_SDK
+      # set ascp executable path
       def ascp_path=(v)
         @path_to_ascp=v
       end
 
+      # filename for ascp with optional extension (Windows)
+      def ascp_filename
+        return 'ascp'+Environment.exe_extension
+      end
+
       # location of SDK files
       def folder=(v)
         @sdk_folder=v
@@ -50,26 +57,30 @@ module Aspera
       # @return the list of installed products in format of product_locations
       def installed_products
         if @found_products.nil?
-          @found_products=product_locations
-          # add sdk as first search path
-          @found_products.unshift({# SDK
+          scan_locations=product_locations.clone
+          # add SDK as first search path
+          scan_locations.unshift({
             :expected =>'SDK',
-            :app_root =>self.folder_path,
+            :app_root =>folder_path,
             :sub_bin =>''
           })
-          @found_products.select! do |pl|
-            next false unless Dir.exist?(pl[:app_root])
-            Log.log.debug("found #{pl[:app_root]}")
-            sub_bin = pl[:sub_bin] || BIN_SUBFOLDER
-            pl[:ascp_path]=File.join(pl[:app_root],sub_bin,'ascp'+Environment.exe_extension)
-            next false unless File.exist?(pl[:ascp_path])
-            product_info_file="#{pl[:app_root]}/#{PRODUCT_INFO}"
+          # search installed products: with ascp
+          @found_products=scan_locations.select! do |item|
+            # skip if not main folder
+            next false unless Dir.exist?(item[:app_root])
+            Log.log.debug("Found #{item[:app_root]}")
+            sub_bin = item[:sub_bin] || BIN_SUBFOLDER
+            item[:ascp_path]=File.join(item[:app_root],sub_bin,ascp_filename)
+            # skip if no ascp
+            next false unless File.exist?(item[:ascp_path])
+            # read info from product info file if present
+            product_info_file="#{item[:app_root]}/#{PRODUCT_INFO}"
             if File.exist?(product_info_file)
-              res_s=XmlSimple.xml_in(File.read(product_info_file),{"ForceArray"=>false})
-              pl[:name]=res_s['name']
-              pl[:version]=res_s['version']
+              res_s=XmlSimple.xml_in(File.read(product_info_file),{'ForceArray'=>false})
+              item[:name]=res_s['name']
+              item[:version]=res_s['version']
             else
-              pl[:name]=pl[:expected]
+              item[:name]=item[:expected]
             end
             true # select this version
           end
@@ -77,6 +88,7 @@ module Aspera
         return @found_products
       end
 
+      # all ascp files (in SDK)
       FILES=[:ascp,:ascp4,:ssh_bypass_key_dsa,:ssh_bypass_key_rsa,:aspera_license,:aspera_conf,:fallback_cert,:fallback_key]
 
       # get path of one resource file of currently activated product
@@ -89,39 +101,33 @@ module Aspera
           # note that there might be a .exe at the end
           file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
         when :ssh_bypass_key_dsa
-          file=File.join(self.folder_path,'aspera_bypass_dsa.pem')
+          file=File.join(folder_path,'aspera_bypass_dsa.pem')
           File.write(file,get_key('dsa',1)) unless File.exist?(file)
           File.chmod(0400,file)
         when :ssh_bypass_key_rsa
-          file=File.join(self.folder_path,'aspera_bypass_rsa.pem')
+          file=File.join(folder_path,'aspera_bypass_rsa.pem')
           File.write(file,get_key('rsa',2)) unless File.exist?(file)
           File.chmod(0400,file)
         when :aspera_license
-          file=File.join(self.folder_path,'aspera-license')
+          file=File.join(folder_path,'aspera-license')
           File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.get_bin(6))}==SIGNATURE==\n#{Base64.strict_encode64(DataRepository.instance.get_bin(7))}")) unless File.exist?(file)
           File.chmod(0400,file)
         when :aspera_conf
-          file=File.join(self.folder_path,'aspera.conf')
+          file=File.join(folder_path,'aspera.conf')
           File.write(file,%Q{<?xml version='1.0' encoding='UTF-8'?>
 <CONF version="2">
 <default>
     <file_system>
-        <storage_rc>
-            <adaptive>
-                true
-            </adaptive>
-        </storage_rc>
         <resume_suffix>.aspera-ckpt</resume_suffix>
         <partial_file_suffix>.partial</partial_file_suffix>
-        <replace_illegal_chars>_</replace_illegal_chars>
     </file_system>
 </default>
 </CONF>
 }) unless File.exist?(file)
           File.chmod(0400,file)
         when :fallback_cert,:fallback_key
-          file_key=File.join(self.folder_path,'aspera_fallback_key.pem')
-          file_cert=File.join(self.folder_path,'aspera_fallback_cert.pem')
+          file_key=File.join(folder_path,'aspera_fallback_key.pem')
+          file_cert=File.join(folder_path,'aspera_fallback_cert.pem')
           if !File.exist?(file_key) or !File.exist?(file_cert)
             require 'openssl'
             # create new self signed certificate for http fallback
@@ -147,7 +153,7 @@ module Aspera
         return file
       end
 
-      # @returns the file path of local connect where API's URI can be read
+      # @return the file path of local connect where API's URI can be read
       def connect_uri
         connect=get_product_folders(PRODUCT_CONNECT)
         folder=File.join(connect[:run_root],VARRUN_SUBFOLDER)
@@ -176,23 +182,67 @@ module Aspera
         return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
       end
 
-      def install_sdk
+      # Check that specified path is ascp and get version
+      def get_ascp_version(ascp_path)
+        raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
+        ascp_version='n/a'
+        raise "error in sdk: no ascp included" if ascp_path.nil?
+        cmd_out=%x{"#{ascp_path}" -A}
+        raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
+        # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
+        m=cmd_out.match(/ascp version (.*)/)
+        ascp_version=m[1] unless m.nil?
+      end
+
+      # download aspera SDK or use local file
+      # extracts ascp binary for current system architecture
+      # @return ascp version (from execution)
+      def install_sdk(sdk_url)
         require 'zip'
         sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
-        Aspera::Rest.new(base_url: SDK_URL).call(operation: 'GET',save_to_file: sdk_zip_path)
+        if sdk_url.start_with?('file:')
+          # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
+          raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
+          sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
+        else
+          redirect_remain=MAX_REDIRECT_SDK
+          begin
+            Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
+          rescue Aspera::RestCallError => e
+            if e.response.is_a?(Net::HTTPRedirection)
+              if redirect_remain > 0
+                redirect_remain-=1
+                sdk_url=e.response['location']
+                retry
+              else
+                raise "Too many redirect"
+              end
+            else
+              raise e
+            end
+          end
+        end
+        # SDK is organized by architecture
         filter="/#{Environment.architecture}/"
         ascp_path=nil
+        sdk_path=folder_path
+        # rename old install
+        if File.exist?(File.join(sdk_path,ascp_filename))
+          Log.log.warn("Previous install exists, renaming.")
+          File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
+        end
         # first ensure license file is here so that ascp invokation for version works
         self.path(:aspera_license)
         self.path(:aspera_conf)
         Zip::File.open(sdk_zip_path) do |zip_file|
           zip_file.each do |entry|
+            # get only specified arch, but not folder, only files
             if entry.name.include?(filter) and !entry.name.end_with?('/')
-              archive_file=File.join(self.folder_path,File.basename(entry.name))
+              archive_file=File.join(sdk_path,File.basename(entry.name))
               File.open(archive_file, 'wb') do |output_stream|
                 IO.copy_stream(entry.get_input_stream, output_stream)
               end
-              if entry.name.include?('ascp')
+              if File.basename(entry.name).eql?(ascp_filename)
                 FileUtils.chmod(0755,archive_file)
                 ascp_path=archive_file
               end
@@ -200,13 +250,8 @@ module Aspera
           end
         end
         File.unlink(sdk_zip_path) rescue nil # Windows may give error
-        ascp_version='n/a'
-        if !ascp_path.nil?
-          # get version from ascp, only after full extract, as windows requires SSL/TLS DLLs
-          m=%x{#{ascp_path} -A}.match(/ascp version (.*)/)
-          ascp_version=m[1] unless m.nil?
-          File.write(File.join(self.folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
-        end
+        ascp_version=get_ascp_version(ascp_path)
+        File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
         return ascp_version
       end
 
@@ -219,16 +264,8 @@ module Aspera
       PRODUCT_INFO='product-info.mf'
       # policy for product selection
       FIRST_FOUND='FIRST'
-      SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
-
-      private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO,:SDK_URL
 
-      # get some specific folder from specific applications: Connect or CLI
-      def get_product_folders(name)
-        found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
-        raise "Product: #{name} not found, please install." if found.empty?
-        return found.first
-      end
+      private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
 
       def initialize
         @path_to_ascp=nil
@@ -236,14 +273,22 @@ module Aspera
         @found_products=nil
       end
 
+      # @return folder paths for specified applications
+      # @param name Connect or CLI
+      def get_product_folders(name)
+        found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
+        raise "Product: #{name} not found, please install." if found.empty?
+        return found.first
+      end
+
+      # @return the path to folder where SDK is installed
       def folder_path
-        raise "undefined path to SDK" if @sdk_folder.nil?
+        raise "Undefined path to SDK" if @sdk_folder.nil?
         FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
         @sdk_folder
       end
 
-      # returns product folders depending on OS
-      # fields
+      # @return product folders depending on OS fields
       # :expected  M app name is taken from the manifest if present, else defaults to this value
       # :app_root  M main folder for the application
       # :log_root  O location of log files (Linux uses syslog)
@@ -285,7 +330,7 @@ module Aspera
             :log_root =>File.join(Dir.home,'Library','Logs','Aspera_Drive'),
             :sub_bin  =>File.join('Contents','Resources'),
             }]
-        else; return [{  # other: Linux and unix family
+        else; return [{  # other: Linux and Unix family
             :expected =>PRODUCT_CONNECT,
             :app_root =>File.join(Dir.home,'.aspera','connect'),
             :run_root =>File.join(Dir.home,'.aspera','connect')
@@ -299,6 +344,9 @@ module Aspera
         end
       end
 
+      # @return a standard bypass key
+      # @param type rsa or dsa
+      # @param id in repository 1 for dsa, 2 for rsa
       def get_key(type,id)
         hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
         bin=Base64.strict_encode64(DataRepository.instance.get_bin(id))

data/lib/aspera/fasp/local.rb

@@ -21,16 +21,24 @@ module Aspera
     ACCESS_KEY_TRANSFER_USER='xfer'
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class Local < Manager
-      ASCP_SPAWN_TIMEOUT_SEC = 3
-      private_constant :ASCP_SPAWN_TIMEOUT_SEC
-      # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
+      # options for initialize
+      DEFAULT_OPTIONS = {
+        :spawn_timeout_sec => 3,
+        :spawn_delay_sec   => 2,
+        :wss               => false,
+        :resume            => {}
+      }
+      DEFAULT_UDP_PORT=33001
+      private_constant :DEFAULT_OPTIONS
+      # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
       attr_accessor :quiet
+
       # start ascp transfer (non blocking), single or multi-session
       # job information added to @jobs
       # @param transfer_spec [Hash] aspera transfer specification
       # @param options [Hash] :resumer, :regenerate_token
       def start_transfer(transfer_spec,options={})
-        raise "option: must be hash (or nil)" unless options.is_a?(Hash)
+        raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
         job_options = options.clone
         job_options[:resumer] ||= @resume_policy
         job_options[:job_id] ||= SecureRandom.uuid
@@ -58,21 +66,26 @@ module Aspera
 
         # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
         # compute this before using transfer spec, even if the var is not used in single session
-        multi_session_udp_port_base=33001
-        multi_session_number=nil
+        multi_session_udp_port_base=DEFAULT_UDP_PORT
+        multi_session_number=0
         if transfer_spec.has_key?('multi_session')
           multi_session_number=transfer_spec['multi_session'].to_i
-          raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
-          # managed here, so delete from transfer spec
-          transfer_spec.delete('multi_session')
-          if transfer_spec.has_key?('fasp_port')
-            multi_session_udp_port_base=transfer_spec['fasp_port']
-            transfer_spec.delete('fasp_port')
+          if multi_session_number < 0
+            Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
+            multi_session_number = 0
+          end
+          if multi_session_number > 0
+            # managed here, so delete from transfer spec
+            transfer_spec.delete('multi_session')
+            if transfer_spec.has_key?('fasp_port')
+              multi_session_udp_port_base=transfer_spec['fasp_port']
+              transfer_spec.delete('fasp_port')
+            end
           end
         end
 
         # compute known args
-        env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
+        env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
 
         # add fallback cert and key as arguments if needed
         if ['1','force'].include?(transfer_spec['http_fallback'])
@@ -98,25 +111,27 @@ module Aspera
           :options  => job_options  # [Hash]
         }
 
-        Log.log.debug("starting session thread(s)")
-        if !multi_session_number
+        if multi_session_number <= 1
+          Log.log.debug('Starting single session thread')
           # single session for transfer : simple
           session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
           xfer_job[:sessions].push(session)
         else
+          Log.log.debug('Starting multi session threads')
           1.upto(multi_session_number) do |i|
+            sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session=session.clone()
             this_session[:env_args]=this_session[:env_args].clone()
             this_session[:env_args][:args]=this_session[:env_args][:args].clone()
             this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
             # necessary only if server is not linux, i.e. server does not support port re-use
-            this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
+            this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
             this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
             xfer_job[:sessions].push(this_session)
           end
         end
-        Log.log.debug("started session thread(s)")
+        Log.log.debug('started session thread(s)')
 
         # add job to list of jobs
         @jobs[job_options[:job_id]]=xfer_job
@@ -128,7 +143,7 @@ module Aspera
       # wait for completion of all jobs started
       # @return list of :success or error message
       def wait_for_transfers_completion
-        Log.log.debug("wait_for_transfers_completion")
+        Log.log.debug('wait_for_transfers_completion')
         # set to non-nil to exit loop
         result=[]
         @jobs.each do |id,job|
@@ -138,7 +153,7 @@ module Aspera
             result.push(session[:error] ? session[:error] : :success)
           end
         end
-        Log.log.debug("all transfers joined")
+        Log.log.debug('all transfers joined')
         # since all are finished and we return the result, clear statuses
         @jobs.clear
         return result
@@ -146,7 +161,7 @@ module Aspera
 
       # used by asession (to be removed ?)
       def shutdown
-        Log.log.debug("fasp local shutdown")
+        Log.log.debug('fasp local shutdown')
       end
 
       # This is the low level method to start the "ascp" process
@@ -158,8 +173,10 @@ module Aspera
       # @param session this session information
       # could be private method
       def start_transfer_with_args_env(env_args,session)
-        raise "env_args must be Hash" unless env_args.is_a?(Hash)
-        raise "session must be Hash" unless session.is_a?(Hash)
+        raise 'env_args must be Hash' unless env_args.is_a?(Hash)
+        raise 'session must be Hash' unless session.is_a?(Hash)
+        # by default we assume an exception will be raised (for ensure block)
+        exception_raised=true
         begin
           Log.log.debug("env_args=#{env_args.inspect}")
           # get location of ascp executable
@@ -182,7 +199,7 @@ module Aspera
           Log.log.debug("before accept for pid (#{ascp_pid})")
           # init management socket
           ascp_mgt_io=nil
-          Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
+          Timeout.timeout(@options[:spawn_timeout_sec]) do
             ascp_mgt_io = mgt_sock.accept
             # management messages include file names which may be utf8
             # by default socket is US-ASCII
@@ -216,7 +233,7 @@ module Aspera
               current_event_data[$1] = $2
             when ''
               # empty line is separator to end event information
-              raise "unexpected empty line" if current_event_data.nil?
+              raise 'unexpected empty line' if current_event_data.nil?
               current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
               notify_listeners(current_event_text,current_event_data)
               case current_event_data['Type']
@@ -232,23 +249,27 @@ module Aspera
             end # case
           end # loop (process mgt port lines)
           # check that last status was received before process exit
-          raise "INTERNAL: nil last status" if last_status_event.nil?
-          case last_status_event['Type']
-          when 'DONE'
-            # return method (or just don't do anything)
-            return
-          when 'ERROR'
-            Log.log.error("code: #{last_status_event['Code']}")
-            if last_status_event['Description']  =~ /bearer token/i
-              Log.log.error("need to regenerate token".red)
-              if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
-                # regenerate token here, expired, or error on it
-                env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
+          if last_status_event.is_a?(Hash)
+            case last_status_event['Type']
+            when 'DONE'
+              # all went well
+              exception_raised=false
+            when 'ERROR'
+              Log.log.error("code: #{last_status_event['Code']}")
+              if last_status_event['Description']  =~ /bearer token/i
+                Log.log.error('need to regenerate token'.red)
+                if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
+                  # regenerate token here, expired, or error on it
+                  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
+                end
               end
+              raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
+            else # case
+              raise "unexpected last event type: #{last_status_event['Type']}"
             end
-            raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
           else
-            raise "unexpected last event type: #{last_status_event['Type']}"
+            exception_raised=false
+            Log.log.debug('no status read from ascp mgt port')
           end
         rescue SystemCallError => e
           # Process.spawn
@@ -262,8 +283,18 @@ module Aspera
           unless ascp_pid.nil?
             # "wait" for process to avoid zombie
             Process.wait(ascp_pid)
+            status=$?
             ascp_pid=nil
             session.delete(:io)
+            if !status.success?
+              message="ascp failed with code #{status.exitstatus}"
+              if exception_raised
+                # just debug, as main exception is already here
+                Log.log.debug(message)
+              else
+                raise Fasp::Error.new(message)
+              end
+            end
           end
         end # begin-ensure
       end # start_transfer_with_args_env
@@ -276,9 +307,9 @@ module Aspera
       # {'type'=>'DONE'}
       def send_command(job_id,session_index,data)
         job=@jobs[job_id]
-        raise "no such job" if job.nil?
+        raise 'no such job' if job.nil?
         session=job[:sessions][session_index]
-        raise "no such session" if session.nil?
+        raise 'no such session' if session.nil?
         Log.log.debug("command: #{data}")
         # build command
         command=data.
@@ -292,8 +323,8 @@ module Aspera
 
       private
 
-      def initialize(agent_options=nil)
-        agent_options||={}
+      # @param options : keys(symbol): wss, resume
+      def initialize(options=nil)
         super()
         # by default no interactive progress bar
         @quiet=true
@@ -301,8 +332,20 @@ module Aspera
         @jobs={}
         # mutex protects global data accessed by threads
         @mutex=Mutex.new
-        @resume_policy=ResumePolicy.new(agent_options)
-        @enable_wss = agent_options[:wss] || false
+        # manage options
+        @options=DEFAULT_OPTIONS.clone
+        if !options.nil?
+          raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
+          options.each do |k,v|
+            if DEFAULT_OPTIONS.has_key?(k)
+              @options[k]=v
+            else
+              raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
+            end
+          end
+        end
+        Log.log.debug("local options= #{options}")
+        @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
       end
 
       # transfer thread entry
@@ -310,7 +353,7 @@ module Aspera
       def transfer_thread_entry(session)
         begin
           # set name for logging
-          Thread.current[:name]="transfer"
+          Thread.current[:name]='transfer'
           Log.log.debug("ENTER (#{Thread.current[:name]})")
           # start transfer with selected resumer policy
           session[:options][:resumer].process do