aspera-cli 4.0.0.pre3 → 4.2.1

data/lib/aspera/fasp/manager.rb

@@ -45,6 +45,18 @@ module Aspera
         end
       end # notify_listeners
 
+      def notify_begin(id,size)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
+      end
+
+      def notify_progress(id,size)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
+      end
+
+      def notify_end(id)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
+      end
+
       public
       LISTENER_SESSION_ID_B='ListenerSessionId'
       LISTENER_SESSION_ID_S='listener_session_id'
@@ -60,6 +72,9 @@ module Aspera
       # start_transfer(transfer_spec,options) : start and wait for completion
       # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
       # optional: shutdown
+      
+      # This checks the validity of the value returned by wait_for_transfers_completion
+      # it must be a list of :success or exception
       def self.validate_status_list(statuses)
         raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
         raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?

data/lib/aspera/fasp/node.rb

@@ -55,7 +55,7 @@ module Aspera
           trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
           case trdata['status']
           when 'completed'
-            notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
+            notify_end(@transfer_id)
             break
           when 'waiting','partially_completed','unknown','waiting(read error)'
             if spinner.nil?
@@ -69,10 +69,10 @@ module Aspera
             #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
             if !started and trdata['precalc'].is_a?(Hash) and
             trdata['precalc']['status'].eql?('ready')
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
+              notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
               started=true
             else
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
+              notify_progress(@transfer_id,trdata['bytes_transferred'])
             end
           else
             Log.log.warn("trdata -> #{trdata}")
@@ -81,7 +81,7 @@ module Aspera
           sleep 1
         end
         #TODO get status of sessions
-        return [] 
+        return []
       end
     end
   end

data/lib/aspera/fasp/parameters.rb

@@ -16,10 +16,6 @@ module Aspera
       # because of garbage collection takes any file there
       # this could be refined, as , for instance, on macos, temp folder is already user specific
       @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-      SEC_IN_DAY=86400
-      # assume no transfer last longer than this
-      # (garbage collect file list which were not deleted after transfer)
-      FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
       PARAM_DEFINITION={
         # parameters with env vars
         'remote_password'         => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
@@ -63,7 +59,7 @@ module Aspera
         'exclude_older_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
         'preserve_acls'           => { :type => :opt_with_arg, :accepted_types=>String},
         'move_after_transfer'     => { :type => :opt_with_arg, :accepted_types=>String},
-        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
+        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>Integer},
         # non standard parameters
         'EX_fasp_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
         'EX_http_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
@@ -82,11 +78,12 @@ module Aspera
         'lock_rate_policy'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         'lock_min_rate'           => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         'lock_target_rate'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        #'authentication'          => { :type => :ignore, :accepted_types=>String}, # = token
+        'authentication'          => { :type => :ignore, :accepted_types=>String}, # value = token
         'https_fallback_port'     => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
         'content_protection'      => { :type => :ignore, :accepted_types=>String},
         'cipher_allowed'          => { :type => :ignore, :accepted_types=>String},
         'multi_session'           => { :type => :ignore, :accepted_types=>Integer}, # managed
+        'obfuscate_file_names'          => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         # optional tags (  additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'}  )
         'tags'                    => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
         # special processing @builder.process_param( called individually
@@ -100,7 +97,7 @@ module Aspera
         'wss_port'                => { :type => :defer, :accepted_types=>Integer},
       }
 
-      private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC,:PARAM_DEFINITION
+      private_constant :PARAM_DEFINITION
 
       def initialize(job_spec,options)
         @job_spec=job_spec
@@ -207,18 +204,9 @@ module Aspera
       # temp file list files are created here
       def self.file_list_folder=(v)
         @@file_list_folder=v
-        unless @@file_list_folder.nil?
+        if !@@file_list_folder.nil?
           FileUtils.mkdir_p(@@file_list_folder)
-          # garbage collect undeleted files
-          Dir.entries(@@file_list_folder).each do |name|
-            file_path=File.join(@@file_list_folder,name)
-            age_sec=(Time.now - File.stat(file_path).mtime).to_i
-            # check age of file, delete too old
-            if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
-              Log.log.debug("garbage collecting #{name}")
-              File.delete(file_path)
-            end
-          end
+          TempFileManager.instance.cleanup_expired(@@file_list_folder)
         end
       end
 

data/lib/aspera/fasp/resume_policy.rb

@@ -14,17 +14,21 @@ module Aspera
         :sleep_max     => 60
       }
 
-      def initialize(params={})
+      # @param params see DEFAULTS
+      def initialize(params=nil)
         @parameters=DEFAULTS.clone
-        return if params.nil?
-        raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
-        params.each do |k,v|
-          if DEFAULTS.has_key?(k)
-            @parameters[k]=v
-          else
-            raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+        if !params.nil?
+          raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
+          params.each do |k,v|
+            if DEFAULTS.has_key?(k)
+              raise "#{k} must be Integer" unless v.is_a?(Integer)
+              @parameters[k]=v
+            else
+              raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+            end
           end
         end
+        Log.log.debug("resume params=#{@parameters}")
       end
 
       # calls block a number of times (resumes) until success or limit reached
@@ -45,10 +49,7 @@ module Aspera
             # failure in ascp
             if e.retryable? then
               # exit if we exceed the max number of retry
-              unless remaining_resumes > 0
-                Log.log.error "Maximum number of retry reached"
-                raise Fasp::Error,"max retry after: [#{status[:message]}]"
-              end
+              raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
             else
               # give one chance only to non retryable errors
               unless remaining_resumes.eql?(@parameters[:iter_max])

data/lib/aspera/log.rb

@@ -14,7 +14,7 @@ module Aspera
     attr_reader :logger
     attr_reader :logger_type
     # levels are :debug,:info,:warn,:error,fatal,:unknown
-    def self.levels; Logger::Severity.constants.map{|c| c.downcase.to_sym};end
+    def self.levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
     # where logs are sent to
     def self.logtypes; [:stderr,:stdout,:syslog];end

data/lib/aspera/node.rb

@@ -1,14 +1,74 @@
+require 'aspera/rest'
+require 'aspera/log'
 require 'zlib'
 require 'base64'
+
 module Aspera
   # Provides additional functions using node API.
   class Node < Rest
+    # permissions
+    ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
+    MATCH_EXEC_PREFIX='exec:'
+
+    # for information only
     def self.decode_bearer_token(token)
       return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
     end
+
+    # for access keys: provide expression to match entry in folder
+    # if no prefix: regex
+    # if prefix: ruby code
+    # if filder is nil, then always match
+    def self.file_matcher(match_expression)
+      match_expression||="#{MATCH_EXEC_PREFIX}true"
+      if match_expression.start_with?(MATCH_EXEC_PREFIX)
+        return eval "lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
+      end
+      return lambda{|f|f['name'].match(/#{match_expression}/)}
+    end
+
     def initialize(rest_params)
       super(rest_params)
-      # specifics here
+    end
+
+    # recursively crawl in a folder.
+    # subfolders a processed if the processing method returns true
+    # @param processor must provide a method to process each entry
+    # @param opt options
+    # - top_file_id file id to start at (default = access key root file id)
+    # - top_file_path path of top folder (default = /)
+    # - method processing method (default= process_entry)
+    def crawl(processor,opt={})
+      Log.log.debug("crawl1 #{opt}")
+      # not possible with bearer token
+      opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
+      opt[:top_file_path] ||= '/'
+      opt[:method] ||= :process_entry
+      raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
+      Log.log.debug("crawl #{opt}")
+      #top_info=read("files/#{opt[:top_file_id]}")[:data]
+      folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
+      Log.dump(:folders_to_explore,folders_to_explore)
+      while !folders_to_explore.empty? do
+        current_item = folders_to_explore.shift
+        Log.log.debug("searching #{current_item[:relpath]}".bg_green)
+        # get folder content
+        folder_contents = begin
+          read("files/#{current_item[:id]}/files")[:data]
+        rescue => e
+          Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
+          []
+        end
+        Log.dump(:folder_contents,folder_contents)
+        folder_contents.each do |entry|
+          relative_path=File.join(current_item[:relpath],entry['name'])
+          Log.log.debug("looking #{relative_path}".bg_green)
+          # entry type is file, folder or link
+          if processor.send(opt[:method],entry,relative_path) and entry['type'].eql?('folder')
+            folders_to_explore.push({:id=>entry['id'],:relpath=>relative_path})
+          end
+        end
+      end
     end
   end
 end

data/lib/aspera/oauth.rb

@@ -1,22 +1,26 @@
 require 'aspera/open_application'
+require 'aspera/web_auth'
 require 'base64'
 require 'date'
 require 'socket'
 require 'securerandom'
 
 module Aspera
-  # implement OAuth 2 for the REST client and generate a bearer token
+  # Implement OAuth 2 for the REST client and generate a bearer token
   # call get_authorization() to get a token.
   # bearer tokens are kept in memory and also in a file cache for later re-use
   # if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
   class Oauth
     private
     # remove 5 minutes to account for time offset (TODO: configurable?)
-    JWT_NOTBEFORE_OFFSET=300
+    JWT_NOTBEFORE_OFFSET_SEC=300
     # one hour validity (TODO: configurable?)
-    JWT_EXPIRY_OFFSET=3600
+    JWT_EXPIRY_OFFSET_SEC=3600
+    # tokens older than 30 minutes will be discarded from cache
+    TOKEN_CACHE_EXPIRY_SEC=1800
+    # a prefix for persistency of tokens (garbage collect)
     PERSIST_CATEGORY_TOKEN='token'
-    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN
+    private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC
     class << self
       # OAuth methods supported
       def auth_types
@@ -28,12 +32,18 @@ module Aspera
       end
 
       def persist_mgr
-        raise "set persistency manager first" if @persist.nil?
+        if @persist.nil?
+          Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
+          # create NULL persistency class
+          @persist=Class.new do
+            def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
+          end.new
+        end
         return @persist
       end
 
       def flush_tokens
-        persist_mgr.flush_by_prefix(PERSIST_CATEGORY_TOKEN)
+        persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
       end
     end
 
@@ -79,13 +89,19 @@ module Aspera
         raise "redirect_uri must have a port" if uri.port.nil?
         # we could check that host is localhost or local address
       end
+      # cleanup expired tokens
+      self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
     end
 
-    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
-
     # open the login page, wait for code and check_code, then return code
     def goto_page_and_get_code(login_page_url,check_code)
-      request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
+      Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
+      # start a web server to receive request code
+      webserver=WebAuth.new(@params[:redirect_uri])
+      # start browser on login page
+      OpenApplication.instance.uri(login_page_url)
+      # wait for code in request
+      request_params=webserver.get_request
       Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
       code=request_params['code']
       return code
@@ -117,6 +133,9 @@ module Aspera
 
     public
 
+    # used to change parameter, such as scope
+    attr_reader :params
+
     # @param options : :scope and :refresh
     def get_authorization(options={})
       # api scope can be overriden to get auth for other scope
@@ -184,14 +203,13 @@ module Aspera
         when :web
           # AoC Web based Auth
           check_code=SecureRandom.uuid
-          login_page_url=Rest.build_uri(
-          "#{@params[:base_url]}/#{@params[:path_authorize]}",
-          p_client_id_and_scope.merge({
+          auth_params=p_client_id_and_scope.merge({
             :response_type => 'code',
             :redirect_uri  => @params[:redirect_uri],
-            :client_secret => @params[:client_secret],
             :state         => check_code
-          }))
+          })
+          auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
+          login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
           # here, we need a human to authorize on a web page
           code=goto_page_and_get_code(login_page_url,check_code)
           # exchange code for token
@@ -211,9 +229,19 @@ module Aspera
             :iss => @params[:client_id],    # issuer
             :sub => @params[:jwt_subject],  # subject
             :aud => @params[:jwt_audience], # audience
-            :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
-            :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
+            :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
+            :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
           }
+          # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
+          if @params[:jwt_is_f5]
+            payload[:jti] = SecureRandom.uuid
+            payload[:iat] = seconds_since_epoch
+            payload.delete(:nbf)
+            p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
+            p_scope[:state]=SecureRandom.uuid
+            p_scope[:client_id]=@params[:client_id]
+            @token_auth_api.params[:auth]={:type=>:none}
+          end
 
           # non standard, only for global ids
           payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
@@ -222,8 +250,8 @@ module Aspera
 
           Log.log.debug("private=[#{rsa_private}]")
 
-          Log.log.debug("JWT assertion=[#{payload}]")
-          assertion = JWT.encode(payload, rsa_private, 'RS256')
+          Log.log.debug("JWT payload=[#{payload}]")
+          assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
 
           Log.log.debug("assertion=[#{assertion}]")
 
@@ -233,8 +261,10 @@ module Aspera
           }))
         when :url_token
           # AoC Public Link
+          params={:url_token=>@params[:url_token]}
+          params[:password]=@params[:password] if @params.has_key?(:password)
           resp=create_token_advanced({
-            :json_params => {:url_token=>@params[:url_token]},
+            :json_params => params,
             :url_params  => p_scope.merge({
             :grant_type    => 'url_token'
             })})
@@ -288,32 +318,5 @@ module Aspera
       return 'Bearer '+token_data[@params[:token_field]]
     end
 
-    # open the login page, wait for code and return parameters
-    def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
-      Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
-      # browser start is not blocking, we hope here that starting is slower than opening port
-      OpenApplication.instance.uri(login_page_url)
-      port=URI.parse(redirect_uri).port
-      Log.log.info "listening on port #{port}"
-      request_params=nil
-      TCPServer.open('127.0.0.1', port) { |webserver|
-        Log.log.info "server=#{webserver}"
-        websession = webserver.accept
-        sleep 1 # TODO: sometimes: returns nil ? use webrick ?
-        line = websession.gets.chomp
-        Log.log.info "line=#{line}"
-        if ! line.start_with?('GET /?') then
-          raise "unexpected request"
-        end
-        request = line.partition('?').last.partition(' ').first
-        data=URI.decode_www_form(request)
-        request_params=data.to_h
-        Log.log.debug "request_params=#{request_params}"
-        websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
-        websession.close
-      }
-      return request_params
-    end
-
   end # OAuth
 end # Aspera

data/lib/aspera/persistency_folder.rb

@@ -55,12 +55,17 @@ module Aspera
       @cache.delete(object_id)
     end
 
-    def flush_by_prefix(persist_category)
-      persist_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
-      persist_files.each do |filepath|
+    def garbage_collect(persist_category,max_age_seconds=nil)
+      garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
+      if !max_age_seconds.nil?
+        current_time = Time.now
+        garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
+      end
+      garbage_files.each do |filepath|
         File.delete(filepath)
+        Log.log.debug("Deleted expired: #{filepath}")
       end
-      return persist_files
+      return garbage_files
     end
 
     private