aspera-cli 4.0.0.pre2 → 4.2.0

data/lib/aspera/fasp/manager.rb

@@ -45,6 +45,18 @@ module Aspera
         end
       end # notify_listeners
 
+      def notify_begin(id,size)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
+      end
+
+      def notify_progress(id,size)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
+      end
+
+      def notify_end(id)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
+      end
+
       public
       LISTENER_SESSION_ID_B='ListenerSessionId'
       LISTENER_SESSION_ID_S='listener_session_id'

data/lib/aspera/fasp/node.rb

@@ -55,7 +55,7 @@ module Aspera
           trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
           case trdata['status']
           when 'completed'
-            notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
+            notify_end(@transfer_id)
             break
           when 'waiting','partially_completed','unknown','waiting(read error)'
             if spinner.nil?
@@ -69,10 +69,10 @@ module Aspera
             #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
             if !started and trdata['precalc'].is_a?(Hash) and
             trdata['precalc']['status'].eql?('ready')
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
+              notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
               started=true
             else
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
+              notify_progress(@transfer_id,trdata['bytes_transferred'])
             end
           else
             Log.log.warn("trdata -> #{trdata}")
@@ -81,7 +81,7 @@ module Aspera
           sleep 1
         end
         #TODO get status of sessions
-        return [] 
+        return []
       end
     end
   end

data/lib/aspera/fasp/parameters.rb

@@ -16,10 +16,6 @@ module Aspera
       # because of garbage collection takes any file there
       # this could be refined, as , for instance, on macos, temp folder is already user specific
       @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-      SEC_IN_DAY=86400
-      # assume no transfer last longer than this
-      # (garbage collect file list which were not deleted after transfer)
-      FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
       PARAM_DEFINITION={
         # parameters with env vars
         'remote_password'         => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
@@ -63,7 +59,7 @@ module Aspera
         'exclude_older_than'      => { :type => :opt_with_arg, :accepted_types=>Integer},
         'preserve_acls'           => { :type => :opt_with_arg, :accepted_types=>String},
         'move_after_transfer'     => { :type => :opt_with_arg, :accepted_types=>String},
-        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
+        'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>Integer},
         # non standard parameters
         'EX_fasp_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
         'EX_http_proxy_url'       => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
@@ -82,11 +78,12 @@ module Aspera
         'lock_rate_policy'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         'lock_min_rate'           => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         'lock_target_rate'        => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
-        #'authentication'          => { :type => :ignore, :accepted_types=>String}, # = token
+        'authentication'          => { :type => :ignore, :accepted_types=>String}, # value = token
         'https_fallback_port'     => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
         'content_protection'      => { :type => :ignore, :accepted_types=>String},
         'cipher_allowed'          => { :type => :ignore, :accepted_types=>String},
         'multi_session'           => { :type => :ignore, :accepted_types=>Integer}, # managed
+        'obfuscate_file_names'          => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
         # optional tags (  additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'}  )
         'tags'                    => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
         # special processing @builder.process_param( called individually
@@ -100,7 +97,7 @@ module Aspera
         'wss_port'                => { :type => :defer, :accepted_types=>Integer},
       }
 
-      private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC,:PARAM_DEFINITION
+      private_constant :PARAM_DEFINITION
 
       def initialize(job_spec,options)
         @job_spec=job_spec
@@ -207,18 +204,9 @@ module Aspera
       # temp file list files are created here
       def self.file_list_folder=(v)
         @@file_list_folder=v
-        unless @@file_list_folder.nil?
+        if !@@file_list_folder.nil?
           FileUtils.mkdir_p(@@file_list_folder)
-          # garbage collect undeleted files
-          Dir.entries(@@file_list_folder).each do |name|
-            file_path=File.join(@@file_list_folder,name)
-            age_sec=(Time.now - File.stat(file_path).mtime).to_i
-            # check age of file, delete too old
-            if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
-              Log.log.debug("garbage collecting #{name}")
-              File.delete(file_path)
-            end
-          end
+          TempFileManager.instance.cleanup_expired(@@file_list_folder)
         end
       end
 

data/lib/aspera/fasp/resume_policy.rb

@@ -14,17 +14,21 @@ module Aspera
         :sleep_max     => 60
       }
 
-      def initialize(params={})
+      # @param params see DEFAULTS
+      def initialize(params=nil)
         @parameters=DEFAULTS.clone
-        return if params.nil?
-        raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
-        params.each do |k,v|
-          if DEFAULTS.has_key?(k)
-            @parameters[k]=v
-          else
-            raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+        if !params.nil?
+          raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
+          params.each do |k,v|
+            if DEFAULTS.has_key?(k)
+              raise "#{k} must be Integer" unless v.is_a?(Integer)
+              @parameters[k]=v
+            else
+              raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+            end
           end
         end
+        Log.log.debug("resume params=#{@parameters}")
       end
 
       # calls block a number of times (resumes) until success or limit reached
@@ -45,10 +49,7 @@ module Aspera
             # failure in ascp
             if e.retryable? then
               # exit if we exceed the max number of retry
-              unless remaining_resumes > 0
-                Log.log.error "Maximum number of retry reached"
-                raise Fasp::Error,"max retry after: [#{status[:message]}]"
-              end
+              raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
             else
               # give one chance only to non retryable errors
               unless remaining_resumes.eql?(@parameters[:iter_max])

data/lib/aspera/log.rb

@@ -14,7 +14,7 @@ module Aspera
     attr_reader :logger
     attr_reader :logger_type
     # levels are :debug,:info,:warn,:error,fatal,:unknown
-    def self.levels; Logger::Severity.constants.map{|c| c.downcase.to_sym};end
+    def self.levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
     # where logs are sent to
     def self.logtypes; [:stderr,:stdout,:syslog];end
@@ -49,7 +49,15 @@ module Aspera
 
     # change underlying logger, but keep log level
     def logger_type=(new_logtype)
-      current_severity_integer=@logger.nil? ? Logger::Severity::WARN : @logger.level
+      current_severity_integer=if @logger.nil?
+        if ENV.has_key?('AS_LOG_LEVEL')
+          ENV['AS_LOG_LEVEL']
+        else
+          Logger::Severity::WARN
+        end
+      else
+        @logger.level
+      end
       case new_logtype
       when :stderr
         @logger = Logger.new(STDERR)

data/lib/aspera/node.rb

@@ -1,14 +1,74 @@
+require 'aspera/rest'
+require 'aspera/log'
 require 'zlib'
 require 'base64'
+
 module Aspera
   # Provides additional functions using node API.
   class Node < Rest
+    # permissions
+    ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
+    MATCH_EXEC_PREFIX='exec:'
+
+    # for information only
     def self.decode_bearer_token(token)
       return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
     end
+
+    # for access keys: provide expression to match entry in folder
+    # if no prefix: regex
+    # if prefix: ruby code
+    # if filder is nil, then always match
+    def self.file_matcher(match_expression)
+      match_expression||="#{MATCH_EXEC_PREFIX}true"
+      if match_expression.start_with?(MATCH_EXEC_PREFIX)
+        return eval "lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
+      end
+      return lambda{|f|f['name'].match(/#{match_expression}/)}
+    end
+
     def initialize(rest_params)
       super(rest_params)
-      # specifics here
+    end
+
+    # recursively crawl in a folder.
+    # subfolders a processed if the processing method returns true
+    # @param processor must provide a method to process each entry
+    # @param opt options
+    # - top_file_id file id to start at (default = access key root file id)
+    # - top_file_path path of top folder (default = /)
+    # - method processing method (default= process_entry)
+    def crawl(processor,opt={})
+      Log.log.debug("crawl1 #{opt}")
+      # not possible with bearer token
+      opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
+      opt[:top_file_path] ||= '/'
+      opt[:method] ||= :process_entry
+      raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
+      Log.log.debug("crawl #{opt}")
+      #top_info=read("files/#{opt[:top_file_id]}")[:data]
+      folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
+      Log.dump(:folders_to_explore,folders_to_explore)
+      while !folders_to_explore.empty? do
+        current_item = folders_to_explore.shift
+        Log.log.debug("searching #{current_item[:relpath]}".bg_green)
+        # get folder content
+        folder_contents = begin
+          read("files/#{current_item[:id]}/files")[:data]
+        rescue => e
+          Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
+          []
+        end
+        Log.dump(:folder_contents,folder_contents)
+        folder_contents.each do |entry|
+          relative_path=File.join(current_item[:relpath],entry['name'])
+          Log.log.debug("looking #{relative_path}".bg_green)
+          # entry type is file, folder or link
+          if processor.send(opt[:method],entry,relative_path) and entry['type'].eql?('folder')
+            folders_to_explore.push({:id=>entry['id'],:relpath=>relative_path})
+          end
+        end
+      end
     end
   end
 end

data/lib/aspera/oauth.rb

@@ -16,7 +16,10 @@ module Aspera
     # one hour validity (TODO: configurable?)
     JWT_EXPIRY_OFFSET=3600
     PERSIST_CATEGORY_TOKEN='token'
-    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN
+    # tokens older than 30 minutes will be discarded
+    TOKEN_EXPIRY_SECONDS=1800
+    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
+    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN,:TOKEN_EXPIRY_SECONDS,:THANK_YOU_HTML
     class << self
       # OAuth methods supported
       def auth_types
@@ -28,12 +31,18 @@ module Aspera
       end
 
       def persist_mgr
-        raise "set persistency manager first" if @persist.nil?
+        if @persist.nil?
+          Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
+          # create NULL persistency class
+          @persist=Class.new do
+            def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
+          end.new
+        end
         return @persist
       end
 
       def flush_tokens
-        persist_mgr.flush_by_prefix(PERSIST_CATEGORY_TOKEN)
+        persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
       end
     end
 
@@ -79,10 +88,10 @@ module Aspera
         raise "redirect_uri must have a port" if uri.port.nil?
         # we could check that host is localhost or local address
       end
+      # cleanup expired tokens
+      self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_EXPIRY_SECONDS)
     end
 
-    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
-
     # open the login page, wait for code and check_code, then return code
     def goto_page_and_get_code(login_page_url,check_code)
       request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
@@ -117,6 +126,9 @@ module Aspera
 
     public
 
+    # used to change parameter, such as scope
+    attr_reader :params
+
     # @param options : :scope and :refresh
     def get_authorization(options={})
       # api scope can be overriden to get auth for other scope
@@ -184,14 +196,13 @@ module Aspera
         when :web
           # AoC Web based Auth
           check_code=SecureRandom.uuid
-          login_page_url=Rest.build_uri(
-          "#{@params[:base_url]}/#{@params[:path_authorize]}",
-          p_client_id_and_scope.merge({
+          auth_params=p_client_id_and_scope.merge({
             :response_type => 'code',
             :redirect_uri  => @params[:redirect_uri],
-            :client_secret => @params[:client_secret],
             :state         => check_code
-          }))
+          })
+          auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
+          login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
           # here, we need a human to authorize on a web page
           code=goto_page_and_get_code(login_page_url,check_code)
           # exchange code for token
@@ -214,6 +225,16 @@ module Aspera
             :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
             :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
           }
+          # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
+          if @params[:jwt_is_f5]
+            payload[:jti] = SecureRandom.uuid
+            payload[:iat] = seconds_since_epoch
+            payload.delete(:nbf)
+            p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
+            p_scope[:state]=SecureRandom.uuid
+            p_scope[:client_id]=@params[:client_id]
+            @token_auth_api.params[:auth]={:type=>:none}
+          end
 
           # non standard, only for global ids
           payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
@@ -222,8 +243,8 @@ module Aspera
 
           Log.log.debug("private=[#{rsa_private}]")
 
-          Log.log.debug("JWT assertion=[#{payload}]")
-          assertion = JWT.encode(payload, rsa_private, 'RS256')
+          Log.log.debug("JWT payload=[#{payload}]")
+          assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
 
           Log.log.debug("assertion=[#{assertion}]")
 
@@ -233,8 +254,10 @@ module Aspera
           }))
         when :url_token
           # AoC Public Link
+          params={:url_token=>@params[:url_token]}
+          params[:password]=@params[:password] if @params.has_key?(:password)
           resp=create_token_advanced({
-            :json_params => {:url_token=>@params[:url_token]},
+            :json_params => params,
             :url_params  => p_scope.merge({
             :grant_type    => 'url_token'
             })})

data/lib/aspera/persistency_folder.rb

@@ -55,12 +55,17 @@ module Aspera
       @cache.delete(object_id)
     end
 
-    def flush_by_prefix(persist_category)
-      persist_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
-      persist_files.each do |filepath|
+    def garbage_collect(persist_category,max_age_seconds=nil)
+      garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
+      if !max_age_seconds.nil?
+        current_time = Time.now
+        garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
+      end
+      garbage_files.each do |filepath|
         File.delete(filepath)
+        Log.log.debug("Deleted expired: #{filepath}")
       end
-      return persist_files
+      return garbage_files
     end
 
     private

data/lib/aspera/preview/file_types.rb

@@ -1,9 +1,11 @@
-require 'mimemagic'
-require 'mimemagic/overlay'
+require 'aspera/log'
+require 'singleton'
 
 module Aspera
   module Preview
+    # function conversion_type returns one of the types: CONVERSION_TYPES
     class FileTypes
+      include Singleton
       # values for conversion_type : input format
       CONVERSION_TYPES=[
         :image,
@@ -148,6 +150,7 @@ module Aspera
         'dif' => :office,
         'divx' => :video,
         'dng' => :image,
+        'docx' => :office,
         'dpx' => :image,
         'epdf' => :image,
         'epi' => :image,
@@ -204,6 +207,7 @@ module Aspera
         'pam' => :image,
         'pcd' => :image,
         'pcds' => :image,
+        'pdf' => :pdf,
         'pef' => :image,
         'picon' => :image,
         'pict' => :image,
@@ -261,39 +265,67 @@ module Aspera
         'x3f' => :image,
         'xcf' => :image,
         'xlk' => :office,
+        'xlsx' => :office,
+        'xls' => :office,
         'ycbcr' => :image,
         'ycbcra' => :image,
         'yuv' => :image,
         'zabw' => :office}
 
-      #private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
+      private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
 
-      def self.mime_from_file(filepath)
+      # @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
+      attr_accessor :use_mimemagic
+
+      def initialize
+        @use_mimemagic=false
+      end
+
+      # use mime magic to find mime type based on file content (magic numbers)
+      def mime_from_file(filepath)
+        # moved here, as mimemagic can cause installation issues
+        require 'mimemagic'
+        require 'mimemagic/version'
+        require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
+        # check magic number inside file (empty string if not found)
         detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
-        detected_mime=MimeMagic.by_path(filepath).to_s if detected_mime.empty?
+        # check extension only
+        if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
+          Log.log.debug("no conversion for #{detected_mime}, trying extension")
+          detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
+        end
         detected_mime=nil if detected_mime.empty?
+        Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
         return detected_mime
       end
 
-      def self.conversion_type(filepath,mimetype,try_local_mime)
-        detected_mime=nil
-        if try_local_mime
+      # return file type, one of enum CONVERSION_TYPES
+      # @param filepath [String] full path to file
+      # @param mimetype [String] provided by node api
+      def conversion_type(filepath,mimetype)
+        Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
+        # 1- get type from provided mime type, using local mapping
+        conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
+        # 2- else, from computed mime type (if available)
+        if conv_type.nil? and @use_mimemagic
           detected_mime=mime_from_file(filepath)
-          if mimetype.eql?(detected_mime)
-            Log.log.debug("matching mime type per magic number")
-          else
-            # note: detected can be nil
-            Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+          if ! detected_mime.nil?
+            conv_type=SUPPORTED_MIME_TYPES[detected_mime]
+            if ! mimetype.nil?
+              if mimetype.eql?(detected_mime)
+                Log.log.debug("matching mime type per magic number")
+              else
+                # note: detected can be nil
+                Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+              end
+            end
           end
         end
-        # 1- get type from provided mime type
-        result=FileTypes::SUPPORTED_MIME_TYPES[mimetype] unless mimetype.nil?
-        # 2- else, from computed mime type
-        result||=FileTypes::SUPPORTED_MIME_TYPES[detected_mime] unless detected_mime.nil?
-        extension = File.extname(filepath).downcase.gsub(/^\./,'')
-        # 3- else, from local extensions
-        result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
-        return result
+        # 3- else, from extensions, using local mapping
+        extension = File.extname(filepath.downcase)[1..-1]
+        conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
+        Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
+        return conv_type
       end
     end
   end