aspera-cli 4.1.0 → 4.2.0

data/lib/aspera/fasp/local.rb

@@ -21,16 +21,24 @@ module Aspera
     ACCESS_KEY_TRANSFER_USER='xfer'
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class Local < Manager
-      ASCP_SPAWN_TIMEOUT_SEC = 3
-      private_constant :ASCP_SPAWN_TIMEOUT_SEC
-      # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
+      # options for initialize
+      DEFAULT_OPTIONS = {
+        :spawn_timeout_sec => 3,
+        :spawn_delay_sec   => 2,
+        :wss               => false,
+        :resume            => {}
+      }
+      DEFAULT_UDP_PORT=33001
+      private_constant :DEFAULT_OPTIONS
+      # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
       attr_accessor :quiet
+
       # start ascp transfer (non blocking), single or multi-session
       # job information added to @jobs
       # @param transfer_spec [Hash] aspera transfer specification
       # @param options [Hash] :resumer, :regenerate_token
       def start_transfer(transfer_spec,options={})
-        raise "option: must be hash (or nil)" unless options.is_a?(Hash)
+        raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
         job_options = options.clone
         job_options[:resumer] ||= @resume_policy
         job_options[:job_id] ||= SecureRandom.uuid
@@ -58,21 +66,26 @@ module Aspera
 
         # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
         # compute this before using transfer spec, even if the var is not used in single session
-        multi_session_udp_port_base=33001
-        multi_session_number=nil
+        multi_session_udp_port_base=DEFAULT_UDP_PORT
+        multi_session_number=0
         if transfer_spec.has_key?('multi_session')
           multi_session_number=transfer_spec['multi_session'].to_i
-          raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
-          # managed here, so delete from transfer spec
-          transfer_spec.delete('multi_session')
-          if transfer_spec.has_key?('fasp_port')
-            multi_session_udp_port_base=transfer_spec['fasp_port']
-            transfer_spec.delete('fasp_port')
+          if multi_session_number < 0
+            Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
+            multi_session_number = 0
+          end
+          if multi_session_number > 0
+            # managed here, so delete from transfer spec
+            transfer_spec.delete('multi_session')
+            if transfer_spec.has_key?('fasp_port')
+              multi_session_udp_port_base=transfer_spec['fasp_port']
+              transfer_spec.delete('fasp_port')
+            end
           end
         end
 
         # compute known args
-        env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
+        env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
 
         # add fallback cert and key as arguments if needed
         if ['1','force'].include?(transfer_spec['http_fallback'])
@@ -98,25 +111,27 @@ module Aspera
           :options  => job_options  # [Hash]
         }
 
-        Log.log.debug("starting session thread(s)")
-        if !multi_session_number
+        if multi_session_number <= 1
+          Log.log.debug('Starting single session thread')
           # single session for transfer : simple
           session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
           xfer_job[:sessions].push(session)
         else
+          Log.log.debug('Starting multi session threads')
           1.upto(multi_session_number) do |i|
+            sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session=session.clone()
             this_session[:env_args]=this_session[:env_args].clone()
             this_session[:env_args][:args]=this_session[:env_args][:args].clone()
             this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
             # necessary only if server is not linux, i.e. server does not support port re-use
-            this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
+            this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
             this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
             xfer_job[:sessions].push(this_session)
           end
         end
-        Log.log.debug("started session thread(s)")
+        Log.log.debug('started session thread(s)')
 
         # add job to list of jobs
         @jobs[job_options[:job_id]]=xfer_job
@@ -128,7 +143,7 @@ module Aspera
       # wait for completion of all jobs started
       # @return list of :success or error message
       def wait_for_transfers_completion
-        Log.log.debug("wait_for_transfers_completion")
+        Log.log.debug('wait_for_transfers_completion')
         # set to non-nil to exit loop
         result=[]
         @jobs.each do |id,job|
@@ -138,7 +153,7 @@ module Aspera
             result.push(session[:error] ? session[:error] : :success)
           end
         end
-        Log.log.debug("all transfers joined")
+        Log.log.debug('all transfers joined')
         # since all are finished and we return the result, clear statuses
         @jobs.clear
         return result
@@ -146,7 +161,7 @@ module Aspera
 
       # used by asession (to be removed ?)
       def shutdown
-        Log.log.debug("fasp local shutdown")
+        Log.log.debug('fasp local shutdown')
       end
 
       # This is the low level method to start the "ascp" process
@@ -158,8 +173,10 @@ module Aspera
       # @param session this session information
       # could be private method
       def start_transfer_with_args_env(env_args,session)
-        raise "env_args must be Hash" unless env_args.is_a?(Hash)
-        raise "session must be Hash" unless session.is_a?(Hash)
+        raise 'env_args must be Hash' unless env_args.is_a?(Hash)
+        raise 'session must be Hash' unless session.is_a?(Hash)
+        # by default we assume an exception will be raised (for ensure block)
+        exception_raised=true
         begin
           Log.log.debug("env_args=#{env_args.inspect}")
           # get location of ascp executable
@@ -182,7 +199,7 @@ module Aspera
           Log.log.debug("before accept for pid (#{ascp_pid})")
           # init management socket
           ascp_mgt_io=nil
-          Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
+          Timeout.timeout(@options[:spawn_timeout_sec]) do
             ascp_mgt_io = mgt_sock.accept
             # management messages include file names which may be utf8
             # by default socket is US-ASCII
@@ -216,7 +233,7 @@ module Aspera
               current_event_data[$1] = $2
             when ''
               # empty line is separator to end event information
-              raise "unexpected empty line" if current_event_data.nil?
+              raise 'unexpected empty line' if current_event_data.nil?
               current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
               notify_listeners(current_event_text,current_event_data)
               case current_event_data['Type']
@@ -232,26 +249,27 @@ module Aspera
             end # case
           end # loop (process mgt port lines)
           # check that last status was received before process exit
-          if last_status_event.nil?
-            Log.log.warn("no status read from ascp mgt port")
-          else
+          if last_status_event.is_a?(Hash)
             case last_status_event['Type']
             when 'DONE'
-              # return method (or just don't do anything)
-              return
+              # all went well
+              exception_raised=false
             when 'ERROR'
               Log.log.error("code: #{last_status_event['Code']}")
               if last_status_event['Description']  =~ /bearer token/i
-                Log.log.error("need to regenerate token".red)
+                Log.log.error('need to regenerate token'.red)
                 if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
                   # regenerate token here, expired, or error on it
                   env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
                 end
               end
               raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
-            else
+            else # case
               raise "unexpected last event type: #{last_status_event['Type']}"
             end
+          else
+            exception_raised=false
+            Log.log.debug('no status read from ascp mgt port')
           end
         rescue SystemCallError => e
           # Process.spawn
@@ -269,7 +287,13 @@ module Aspera
             ascp_pid=nil
             session.delete(:io)
             if !status.success?
-              raise Fasp::Error.new("ascp failed with code #{status.exitstatus}")
+              message="ascp failed with code #{status.exitstatus}"
+              if exception_raised
+                # just debug, as main exception is already here
+                Log.log.debug(message)
+              else
+                raise Fasp::Error.new(message)
+              end
             end
           end
         end # begin-ensure
@@ -283,9 +307,9 @@ module Aspera
       # {'type'=>'DONE'}
       def send_command(job_id,session_index,data)
         job=@jobs[job_id]
-        raise "no such job" if job.nil?
+        raise 'no such job' if job.nil?
         session=job[:sessions][session_index]
-        raise "no such session" if session.nil?
+        raise 'no such session' if session.nil?
         Log.log.debug("command: #{data}")
         # build command
         command=data.
@@ -299,8 +323,8 @@ module Aspera
 
       private
 
-      def initialize(agent_options=nil)
-        agent_options||={}
+      # @param options : keys(symbol): wss, resume
+      def initialize(options=nil)
         super()
         # by default no interactive progress bar
         @quiet=true
@@ -308,9 +332,20 @@ module Aspera
         @jobs={}
         # mutex protects global data accessed by threads
         @mutex=Mutex.new
-        @enable_wss = agent_options[:wss] || false
-        agent_options.delete(:wss)
-        @resume_policy=ResumePolicy.new(agent_options)
+        # manage options
+        @options=DEFAULT_OPTIONS.clone
+        if !options.nil?
+          raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
+          options.each do |k,v|
+            if DEFAULT_OPTIONS.has_key?(k)
+              @options[k]=v
+            else
+              raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
+            end
+          end
+        end
+        Log.log.debug("local options= #{options}")
+        @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
       end
 
       # transfer thread entry
@@ -318,7 +353,7 @@ module Aspera
       def transfer_thread_entry(session)
         begin
           # set name for logging
-          Thread.current[:name]="transfer"
+          Thread.current[:name]='transfer'
           Log.log.debug("ENTER (#{Thread.current[:name]})")
           # start transfer with selected resumer policy
           session[:options][:resumer].process do

data/lib/aspera/fasp/parameters.rb

@@ -59,7 +59,7 @@ module Aspera
         'exclude_older_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
         'preserve_acls'           => { :type => :opt_with_arg, :accepted_types=>String},
         'move_after_transfer'     => { :type => :opt_with_arg, :accepted_types=>String},
-        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
+        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>Integer},
         # non standard parameters
         'EX_fasp_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
         'EX_http_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
@@ -78,11 +78,12 @@ module Aspera
         'lock_rate_policy'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         'lock_min_rate'           => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         'lock_target_rate'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        #'authentication'          => { :type => :ignore, :accepted_types=>String}, # = token
+        'authentication'          => { :type => :ignore, :accepted_types=>String}, # value = token
         'https_fallback_port'     => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
         'content_protection'      => { :type => :ignore, :accepted_types=>String},
         'cipher_allowed'          => { :type => :ignore, :accepted_types=>String},
         'multi_session'           => { :type => :ignore, :accepted_types=>Integer}, # managed
+        'obfuscate_file_names'          => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         # optional tags (  additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'}  )
         'tags'                    => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
         # special processing @builder.process_param( called individually

data/lib/aspera/fasp/resume_policy.rb

@@ -14,17 +14,21 @@ module Aspera
         :sleep_max     => 60
       }
 
-      def initialize(params={})
+      # @param params see DEFAULTS
+      def initialize(params=nil)
         @parameters=DEFAULTS.clone
-        return if params.nil?
-        raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
-        params.each do |k,v|
-          if DEFAULTS.has_key?(k)
-            @parameters[k]=v
-          else
-            raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+        if !params.nil?
+          raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
+          params.each do |k,v|
+            if DEFAULTS.has_key?(k)
+              raise "#{k} must be Integer" unless v.is_a?(Integer)
+              @parameters[k]=v
+            else
+              raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+            end
           end
         end
+        Log.log.debug("resume params=#{@parameters}")
       end
 
       # calls block a number of times (resumes) until success or limit reached
@@ -45,10 +49,7 @@ module Aspera
             # failure in ascp
             if e.retryable? then
               # exit if we exceed the max number of retry
-              unless remaining_resumes > 0
-                Log.log.error "Maximum number of retry reached"
-                raise Fasp::Error,"max retry after: [#{status[:message]}]"
-              end
+              raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
             else
               # give one chance only to non retryable errors
               unless remaining_resumes.eql?(@parameters[:iter_max])

data/lib/aspera/node.rb

@@ -8,12 +8,25 @@ module Aspera
   class Node < Rest
     # permissions
     ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
+    MATCH_EXEC_PREFIX='exec:'
 
     # for information only
     def self.decode_bearer_token(token)
       return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
     end
 
+    # for access keys: provide expression to match entry in folder
+    # if no prefix: regex
+    # if prefix: ruby code
+    # if filder is nil, then always match
+    def self.file_matcher(match_expression)
+      match_expression||="#{MATCH_EXEC_PREFIX}true"
+      if match_expression.start_with?(MATCH_EXEC_PREFIX)
+        return eval "lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
+      end
+      return lambda{|f|f['name'].match(/#{match_expression}/)}
+    end
+
     def initialize(rest_params)
       super(rest_params)
     end

data/lib/aspera/oauth.rb

@@ -225,6 +225,16 @@ module Aspera
             :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
             :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
           }
+          # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
+          if @params[:jwt_is_f5]
+            payload[:jti] = SecureRandom.uuid
+            payload[:iat] = seconds_since_epoch
+            payload.delete(:nbf)
+            p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
+            p_scope[:state]=SecureRandom.uuid
+            p_scope[:client_id]=@params[:client_id]
+            @token_auth_api.params[:auth]={:type=>:none}
+          end
 
           # non standard, only for global ids
           payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
@@ -233,8 +243,8 @@ module Aspera
 
           Log.log.debug("private=[#{rsa_private}]")
 
-          Log.log.debug("JWT assertion=[#{payload}]")
-          assertion = JWT.encode(payload, rsa_private, 'RS256')
+          Log.log.debug("JWT payload=[#{payload}]")
+          assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
 
           Log.log.debug("assertion=[#{assertion}]")
 

data/lib/aspera/rest.rb

@@ -120,16 +120,6 @@ module Aspera
       # default is no auth
       @params[:auth]||={:type=>:none}
       @params[:not_auth_codes]||=['401']
-      # translate old auth parameters, remove prefix, place in auth (TODO: delete this)
-#      [:auth,:basic,:oauth].each do |p_sym|
-#        p_str=p_sym.to_s+'_'
-#        @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
-#          name=k_sym.to_s[p_str.length..-1]
-#          name='grant' if k_sym.eql?(:oauth_type)
-#          @params[:auth][name.to_sym]=@params[k_sym]
-#          @params.delete(k_sym)
-#        end
-#      end
       @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
       Log.dump('REST params(2)',@params)
     end
@@ -156,6 +146,7 @@ module Aspera
       Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
       call_data[:headers]||={}
       call_data[:headers]['User-Agent'] ||= @@user_agent
+      # defaults from @params are overriden by call dataz
       call_data=@params.deep_merge(call_data)
       case call_data[:auth][:type]
       when :none
@@ -278,7 +269,7 @@ module Aspera
         if e.response.is_a?(Net::HTTPRedirection)
           if tries_remain_redirect > 0
             tries_remain_redirect-=1
-            Log.log.error("URL is moved, check your config: #{e.response['location']}")
+            Log.log.info("URL is moved: #{e.response['location']}")
             raise e
             # TODO: rebuild request with new location
             #retry

data/lib/aspera/secrets.rb

@@ -0,0 +1,20 @@
+module Aspera
+  # Manage secrets in CLI using secure way (encryption, wallet, etc...)
+  class Secrets
+    attr_accessor :default_secret,:all_secrets
+    def initialize()
+      @default_secret=nil
+      @all_secrets={}
+    end
+
+    def get_secret(id=nil,mandatory=true)
+      secret=@default_secret || @all_secrets[id]
+      raise "please provide secret for #{id}" if secret.nil? and mandatory
+      return secret
+    end
+
+    def get_secrets
+      return @all_secrets
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Laurent Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: xml-simple
@@ -293,6 +293,7 @@ files:
 - lib/aspera/rest_call_error.rb
 - lib/aspera/rest_error_analyzer.rb
 - lib/aspera/rest_errors_aspera.rb
+- lib/aspera/secrets.rb
 - lib/aspera/ssh.rb
 - lib/aspera/sync.rb
 - lib/aspera/temp_file_manager.rb