RubyGems - aspera-cli - Versions diffs - 4.0.0 → 4.2.2 - Mend

aspera-cli 4.0.0 → 4.2.2

Files changed (50) hide show

checksums.yaml +4 -4
data/README.md +843 -304
data/bin/dascli +13 -0
data/docs/Makefile +4 -4
data/docs/README.erb.md +805 -172
data/docs/test_env.conf +22 -3
data/examples/aoc.rb +14 -3
data/examples/faspex4.rb +89 -0
data/lib/aspera/aoc.rb +87 -108
data/lib/aspera/cli/formater.rb +2 -0
data/lib/aspera/cli/main.rb +89 -49
data/lib/aspera/cli/plugin.rb +9 -4
data/lib/aspera/cli/plugins/alee.rb +1 -1
data/lib/aspera/cli/plugins/aoc.rb +188 -173
data/lib/aspera/cli/plugins/ats.rb +2 -2
data/lib/aspera/cli/plugins/config.rb +218 -145
data/lib/aspera/cli/plugins/console.rb +2 -2
data/lib/aspera/cli/plugins/faspex.rb +114 -61
data/lib/aspera/cli/plugins/faspex5.rb +85 -43
data/lib/aspera/cli/plugins/node.rb +3 -3
data/lib/aspera/cli/plugins/preview.rb +59 -45
data/lib/aspera/cli/plugins/server.rb +23 -8
data/lib/aspera/cli/transfer_agent.rb +77 -49
data/lib/aspera/cli/version.rb +1 -1
data/lib/aspera/command_line_builder.rb +49 -31
data/lib/aspera/cos_node.rb +33 -28
data/lib/aspera/environment.rb +2 -2
data/lib/aspera/fasp/connect.rb +28 -21
data/lib/aspera/fasp/http_gw.rb +140 -28
data/lib/aspera/fasp/installation.rb +93 -46
data/lib/aspera/fasp/local.rb +88 -45
data/lib/aspera/fasp/manager.rb +15 -0
data/lib/aspera/fasp/node.rb +4 -4
data/lib/aspera/fasp/parameters.rb +59 -101
data/lib/aspera/fasp/parameters.yaml +531 -0
data/lib/aspera/fasp/resume_policy.rb +13 -12
data/lib/aspera/fasp/uri.rb +1 -1
data/lib/aspera/log.rb +1 -1
data/lib/aspera/node.rb +61 -1
data/lib/aspera/oauth.rb +49 -46
data/lib/aspera/persistency_folder.rb +9 -4
data/lib/aspera/preview/file_types.rb +53 -21
data/lib/aspera/preview/generator.rb +3 -3
data/lib/aspera/rest.rb +29 -18
data/lib/aspera/secrets.rb +20 -0
data/lib/aspera/sync.rb +40 -35
data/lib/aspera/temp_file_manager.rb +19 -0
data/lib/aspera/web_auth.rb +105 -0
metadata +54 -20
data/docs/transfer_spec.html +0 -99

data/lib/aspera/oauth.rb CHANGED Viewed

@@ -1,22 +1,26 @@
 require 'aspera/open_application'
+require 'aspera/web_auth'
 require 'base64'
 require 'date'
 require 'socket'
 require 'securerandom'
 module Aspera
-  # implement OAuth 2 for the REST client and generate a bearer token
+  # Implement OAuth 2 for the REST client and generate a bearer token
   # call get_authorization() to get a token.
   # bearer tokens are kept in memory and also in a file cache for later re-use
   # if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
   class Oauth
     private
     # remove 5 minutes to account for time offset (TODO: configurable?)
-    JWT_NOTBEFORE_OFFSET=300
+    JWT_NOTBEFORE_OFFSET_SEC=300
     # one hour validity (TODO: configurable?)
-    JWT_EXPIRY_OFFSET=3600
+    JWT_EXPIRY_OFFSET_SEC=3600
+    # tokens older than 30 minutes will be discarded from cache
+    TOKEN_CACHE_EXPIRY_SEC=1800
+    # a prefix for persistency of tokens (garbage collect)
     PERSIST_CATEGORY_TOKEN='token'
-    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN
+    private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC
     class << self
       # OAuth methods supported
       def auth_types
@@ -28,12 +32,18 @@ module Aspera
       end
       def persist_mgr
-        raise "set persistency manager first" if @persist.nil?
+        if @persist.nil?
+          Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
+          # create NULL persistency class
+          @persist=Class.new do
+            def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
+          end.new
+        end
         return @persist
       end
       def flush_tokens
-        persist_mgr.flush_by_prefix(PERSIST_CATEGORY_TOKEN)
+        persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
       end
     end
@@ -79,13 +89,19 @@ module Aspera
         raise "redirect_uri must have a port" if uri.port.nil?
         # we could check that host is localhost or local address
       end
+      # cleanup expired tokens
+      self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
     end
-    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
     # open the login page, wait for code and check_code, then return code
     def goto_page_and_get_code(login_page_url,check_code)
-      request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
+      Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
+      # start a web server to receive request code
+      webserver=WebAuth.new(@params[:redirect_uri])
+      # start browser on login page
+      OpenApplication.instance.uri(login_page_url)
+      # wait for code in request
+      request_params=webserver.get_request
       Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
       code=request_params['code']
       return code
@@ -117,6 +133,9 @@ module Aspera
     public
+    # used to change parameter, such as scope
+    attr_reader :params
     # @param options : :scope and :refresh
     def get_authorization(options={})
       # api scope can be overriden to get auth for other scope
@@ -184,14 +203,13 @@ module Aspera
         when :web
           # AoC Web based Auth
           check_code=SecureRandom.uuid
-          login_page_url=Rest.build_uri(
-          "#{@params[:base_url]}/#{@params[:path_authorize]}",
-          p_client_id_and_scope.merge({
+          auth_params=p_client_id_and_scope.merge({
             :response_type => 'code',
             :redirect_uri  => @params[:redirect_uri],
-            :client_secret => @params[:client_secret],
             :state         => check_code
-          }))
+          })
+          auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
+          login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
           # here, we need a human to authorize on a web page
           code=goto_page_and_get_code(login_page_url,check_code)
           # exchange code for token
@@ -211,9 +229,19 @@ module Aspera
             :iss => @params[:client_id],    # issuer
             :sub => @params[:jwt_subject],  # subject
             :aud => @params[:jwt_audience], # audience
-            :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
-            :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
+            :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
+            :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
           }
+          # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
+          if @params.has_key?(:f5_username)
+            payload[:jti] = SecureRandom.uuid # JWT id
+            payload[:iat] = seconds_since_epoch # issued at
+            payload.delete(:nbf)
+            p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
+            p_scope[:state]=SecureRandom.uuid
+            p_scope[:client_id]=@params[:client_id]
+            @token_auth_api.params[:auth]={type: :basic,username: @params[:f5_username], password: @params[:f5_password]}
+          end
           # non standard, only for global ids
           payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
@@ -222,8 +250,8 @@ module Aspera
           Log.log.debug("private=[#{rsa_private}]")
-          Log.log.debug("JWT assertion=[#{payload}]")
-          assertion = JWT.encode(payload, rsa_private, 'RS256')
+          Log.log.debug("JWT payload=[#{payload}]")
+          assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
           Log.log.debug("assertion=[#{assertion}]")
@@ -233,8 +261,10 @@ module Aspera
           }))
         when :url_token
           # AoC Public Link
+          params={:url_token=>@params[:url_token]}
+          params[:password]=@params[:password] if @params.has_key?(:password)
           resp=create_token_advanced({
-            :json_params => {:url_token=>@params[:url_token]},
+            :json_params => params,
             :url_params  => p_scope.merge({
             :grant_type    => 'url_token'
             })})
@@ -288,32 +318,5 @@ module Aspera
       return 'Bearer '+token_data[@params[:token_field]]
     end
-    # open the login page, wait for code and return parameters
-    def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
-      Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
-      # browser start is not blocking, we hope here that starting is slower than opening port
-      OpenApplication.instance.uri(login_page_url)
-      port=URI.parse(redirect_uri).port
-      Log.log.info "listening on port #{port}"
-      request_params=nil
-      TCPServer.open('127.0.0.1', port) { |webserver|
-        Log.log.info "server=#{webserver}"
-        websession = webserver.accept
-        sleep 1 # TODO: sometimes: returns nil ? use webrick ?
-        line = websession.gets.chomp
-        Log.log.info "line=#{line}"
-        if ! line.start_with?('GET /?') then
-          raise "unexpected request"
-        end
-        request = line.partition('?').last.partition(' ').first
-        data=URI.decode_www_form(request)
-        request_params=data.to_h
-        Log.log.debug "request_params=#{request_params}"
-        websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
-        websession.close
-      }
-      return request_params
-    end
   end # OAuth
 end # Aspera

data/lib/aspera/persistency_folder.rb CHANGED Viewed

@@ -55,12 +55,17 @@ module Aspera
       @cache.delete(object_id)
     end
-    def flush_by_prefix(persist_category)
-      persist_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
-      persist_files.each do |filepath|
+    def garbage_collect(persist_category,max_age_seconds=nil)
+      garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
+      if !max_age_seconds.nil?
+        current_time = Time.now
+        garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
+      end
+      garbage_files.each do |filepath|
         File.delete(filepath)
+        Log.log.debug("Deleted expired: #{filepath}")
       end
-      return persist_files
+      return garbage_files
     end
     private

data/lib/aspera/preview/file_types.rb CHANGED Viewed

@@ -1,9 +1,11 @@
-require 'mimemagic'
-require 'mimemagic/overlay'
+require 'aspera/log'
+require 'singleton'
 module Aspera
   module Preview
+    # function conversion_type returns one of the types: CONVERSION_TYPES
     class FileTypes
+      include Singleton
       # values for conversion_type : input format
       CONVERSION_TYPES=[
         :image,
@@ -148,6 +150,7 @@ module Aspera
         'dif' => :office,
         'divx' => :video,
         'dng' => :image,
+        'docx' => :office,
         'dpx' => :image,
         'epdf' => :image,
         'epi' => :image,
@@ -204,6 +207,7 @@ module Aspera
         'pam' => :image,
         'pcd' => :image,
         'pcds' => :image,
+        'pdf' => :pdf,
         'pef' => :image,
         'picon' => :image,
         'pict' => :image,
@@ -261,39 +265,67 @@ module Aspera
         'x3f' => :image,
         'xcf' => :image,
         'xlk' => :office,
+        'xlsx' => :office,
+        'xls' => :office,
         'ycbcr' => :image,
         'ycbcra' => :image,
         'yuv' => :image,
         'zabw' => :office}
-      #private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
+      private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
-      def self.mime_from_file(filepath)
+      # @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
+      attr_accessor :use_mimemagic
+      def initialize
+        @use_mimemagic=false
+      end
+      # use mime magic to find mime type based on file content (magic numbers)
+      def mime_from_file(filepath)
+        # moved here, as mimemagic can cause installation issues
+        require 'mimemagic'
+        require 'mimemagic/version'
+        require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
+        # check magic number inside file (empty string if not found)
         detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
-        detected_mime=MimeMagic.by_path(filepath).to_s if detected_mime.empty?
+        # check extension only
+        if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
+          Log.log.debug("no conversion for #{detected_mime}, trying extension")
+          detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
+        end
         detected_mime=nil if detected_mime.empty?
+        Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
         return detected_mime
       end
-      def self.conversion_type(filepath,mimetype,try_local_mime)
-        detected_mime=nil
-        if try_local_mime
+      # return file type, one of enum CONVERSION_TYPES
+      # @param filepath [String] full path to file
+      # @param mimetype [String] provided by node api
+      def conversion_type(filepath,mimetype)
+        Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
+        # 1- get type from provided mime type, using local mapping
+        conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
+        # 2- else, from computed mime type (if available)
+        if conv_type.nil? and @use_mimemagic
           detected_mime=mime_from_file(filepath)
-          if mimetype.eql?(detected_mime)
-            Log.log.debug("matching mime type per magic number")
-          else
-            # note: detected can be nil
-            Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+          if ! detected_mime.nil?
+            conv_type=SUPPORTED_MIME_TYPES[detected_mime]
+            if ! mimetype.nil?
+              if mimetype.eql?(detected_mime)
+                Log.log.debug("matching mime type per magic number")
+              else
+                # note: detected can be nil
+                Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+              end
+            end
           end
         end
-        # 1- get type from provided mime type
-        result=FileTypes::SUPPORTED_MIME_TYPES[mimetype] unless mimetype.nil?
-        # 2- else, from computed mime type
-        result||=FileTypes::SUPPORTED_MIME_TYPES[detected_mime] unless detected_mime.nil?
-        extension = File.extname(filepath).downcase.gsub(/^\./,'')
-        # 3- else, from local extensions
-        result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
-        return result
+        # 3- else, from extensions, using local mapping
+        extension = File.extname(filepath.downcase)[1..-1]
+        conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
+        Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
+        return conv_type
       end
     end
   end

data/lib/aspera/preview/generator.rb CHANGED Viewed

@@ -15,7 +15,7 @@ module Aspera
       # @param src source file path
       # @param dst destination file path
-      # @param api_mime_type optional mime type as provided by node api
+      # @param api_mime_type optional mime type as provided by node api (or nil)
       # node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
       # supported preview type is one of Preview::PREVIEW_FORMATS
       # the resulting preview file type is taken from destination file extension.
@@ -25,14 +25,14 @@ module Aspera
       #   -> preview_format is one of Generator::PREVIEW_FORMATS
       # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
       #  -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
-      def initialize(options,src,dst,main_temp_dir,api_mime_type=nil,try_local_mime=true)
+      def initialize(options,src,dst,main_temp_dir,api_mime_type)
         @options=options
         @source_file_path=src
         @destination_file_path=dst
         @temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
         # extract preview format from extension of target file
         @preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
-        @conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type,try_local_mime)
+        @conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
       end
       # name of processing method in this object

data/lib/aspera/rest.rb CHANGED Viewed

@@ -7,6 +7,7 @@ require 'net/http'
 require 'net/https'
 require 'json'
 require 'base64'
+require 'cgi'
 require 'ruby-progressbar'
 # add cancel method to http
@@ -33,6 +34,7 @@ module Aspera
     # true if https ignore certificate
     @@insecure=false
     @@user_agent='Ruby'
+    @@download_partial_suffix='.http_partial'
     public
     def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
@@ -98,9 +100,10 @@ module Aspera
     public
     attr_reader :params
+    attr_reader :oauth
-    # @param a_rest_params default call parameters and authentication (:auth) :
-    # :type (:basic, :oauth2, :url)
+    # @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
+    # :type (:basic, :oauth2, :url, :none)
     # :username   [:basic]
     # :password   [:basic]
     # :url_creds  [:url]
@@ -117,16 +120,6 @@ module Aspera
       # default is no auth
       @params[:auth]||={:type=>:none}
       @params[:not_auth_codes]||=['401']
-      # translate old auth parameters, remove prefix, place in auth
-      [:auth,:basic,:oauth].each do |p_sym|
-        p_str=p_sym.to_s+'_'
-        @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
-          name=k_sym.to_s[p_str.length..-1]
-          name='grant' if k_sym.eql?(:oauth_type)
-          @params[:auth][name.to_sym]=@params[k_sym]
-          @params.delete(k_sym)
-        end
-      end
       @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
       Log.dump('REST params(2)',@params)
     end
@@ -153,6 +146,7 @@ module Aspera
       Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
       call_data[:headers]||={}
       call_data[:headers]['User-Agent'] ||= @@user_agent
+      # defaults from @params are overriden by call dataz
       call_data=@params.deep_merge(call_data)
       case call_data[:auth][:type]
       when :none
@@ -206,10 +200,13 @@ module Aspera
       Log.log.debug("call_data = #{call_data}")
       result={:http=>nil}
+      # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
+        tries_remain_redirect||=4
         Log.log.debug("send request")
+        # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
           if call_data.has_key?(:save_to_file)
@@ -223,12 +220,12 @@ module Aspera
             target_file=call_data[:save_to_file]
             # override user's path to path in header
             if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
-              target_file=m[1]
+              target_file=File.join(File.dirname(target_file),m[1])
             end
             # download with temp filename
-            target_file_tmp=target_file+'.http.partial'
+            target_file_tmp="#{target_file}#{@@download_partial_suffix}"
             Log.log.debug("saving to: #{target_file}")
-            File.open(target_file_tmp, "wb") do |file|
+            File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
                 new_process=progress.progress+fragment.length
@@ -239,7 +236,7 @@ module Aspera
             # rename at the end
             File.rename(target_file_tmp, target_file)
             progress=nil
-          end
+          end # save_to_file
         end
         # sometimes there is a ITF8 char (e.g. (c) )
         result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
@@ -267,10 +264,24 @@ module Aspera
           end
           Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
           retry unless (oauth_tries -= 1).zero?
-        end # if
+        end # if oauth
+        # moved ?
+        if e.response.is_a?(Net::HTTPRedirection)
+          if tries_remain_redirect > 0
+            tries_remain_redirect-=1
+            Log.log.info("URL is moved: #{e.response['location']}")
+            raise e
+            # TODO: rebuild request with new location
+            #retry
+          else
+            raise "too many redirect"
+          end
+        else
+          raise e
+        end
         # raise exception if could not retry and not return error in result
         raise e unless call_data[:return_error]
-      end
+      end # begin request
       Log.log.debug("result=#{result}")
       return result

data/lib/aspera/secrets.rb ADDED Viewed

@@ -0,0 +1,20 @@
+module Aspera
+  # Manage secrets in CLI using secure way (encryption, wallet, etc...)
+  class Secrets
+    attr_accessor :default_secret,:all_secrets
+    def initialize()
+      @default_secret=nil
+      @all_secrets={}
+    end
+    def get_secret(id=nil,mandatory=true)
+      secret=@default_secret || @all_secrets[id]
+      raise "please provide secret for #{id}" if secret.nil? and mandatory
+      return secret
+    end
+    def get_secrets
+      return @all_secrets
+    end
+  end
+end

data/lib/aspera/sync.rb CHANGED Viewed

@@ -1,48 +1,53 @@
+require 'aspera/command_line_builder'
 module Aspera
   # builds command line arg for async
   class Sync
-    private
     INSTANCE_PARAMS=
     {
-      'alt_logdir'           => { :type => :opt_with_arg, :accepted_types=>String},
-      'watchd'               => { :type => :opt_with_arg, :accepted_types=>String},
-      'apply_local_docroot'  => { :type => :opt_without_arg},
-      'quiet'                => { :type => :opt_without_arg},
+      'alt_logdir'           => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'watchd'               => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'apply_local_docroot'  => { :cltype => :opt_without_arg},
+      'quiet'                => { :cltype => :opt_without_arg},
     }
     SESSION_PARAMS=
     {
-      'name'                 => { :type => :opt_with_arg, :accepted_types=>String},
-      'local_dir'            => { :type => :opt_with_arg, :accepted_types=>String},
-      'remote_dir'           => { :type => :opt_with_arg, :accepted_types=>String},
-      'local_db_dir'         => { :type => :opt_with_arg, :accepted_types=>String},
-      'remote_db_dir'        => { :type => :opt_with_arg, :accepted_types=>String},
-      'host'                 => { :type => :opt_with_arg, :accepted_types=>String},
-      'user'                 => { :type => :opt_with_arg, :accepted_types=>String},
-      'private_key_path'     => { :type => :opt_with_arg, :accepted_types=>String},
-      'direction'            => { :type => :opt_with_arg, :accepted_types=>String},
-      'checksum'             => { :type => :opt_with_arg, :accepted_types=>String},
-      'tcp_port'             => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'rate_policy'          => { :type => :opt_with_arg, :accepted_types=>String},
-      'target_rate'          => { :type => :opt_with_arg, :accepted_types=>String},
-      'cooloff'              => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'pending_max'          => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'scan_intensity'       => { :type => :opt_with_arg, :accepted_types=>String},
-      'cipher'               => { :type => :opt_with_arg, :accepted_types=>String},
-      'transfer_threads'     => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'preserve_time'        => { :type => :opt_without_arg},
-      'preserve_access_time' => { :type => :opt_without_arg},
-      'preserve_modification_time' => { :type => :opt_without_arg},
-      'preserve_uid'         => { :type => :opt_without_arg},
-      'preserve_gid'         => { :type => :opt_without_arg},
-      'create_dir'           => { :type => :opt_without_arg},
-      'reset'                => { :type => :opt_without_arg},
+      'name'                 => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'local_dir'            => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'remote_dir'           => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'local_db_dir'         => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'remote_db_dir'        => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'host'                 => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'user'                 => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'private_key_path'     => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'direction'            => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'checksum'             => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'tcp_port'             => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'rate_policy'          => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'target_rate'          => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'cooloff'              => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'pending_max'          => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'scan_intensity'       => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'cipher'               => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'transfer_threads'     => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'preserve_time'        => { :cltype => :opt_without_arg},
+      'preserve_access_time' => { :cltype => :opt_without_arg},
+      'preserve_modification_time' => { :cltype => :opt_without_arg},
+      'preserve_uid'         => { :cltype => :opt_without_arg},
+      'preserve_gid'         => { :cltype => :opt_without_arg},
+      'create_dir'           => { :cltype => :opt_without_arg},
+      'reset'                => { :cltype => :opt_without_arg},
       # note: only one env var, but multiple sessions... may be a problem
-      'remote_password'      => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
-      'cookie'               => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
-      'token'                => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
-      'license'              => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
+      'remote_password'      => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_PASS'},
+      'cookie'               => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_COOKIE'},
+      'token'                => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_TOKEN'},
+      'license'              => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_LICENSE'},
     }
-    public
+    Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
+    Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
+    private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
     def initialize(sync_params)
       @sync_params=sync_params

data/lib/aspera/temp_file_manager.rb CHANGED Viewed

@@ -6,6 +6,11 @@ module Aspera
   # create a temp file name for a given folder
   # files can be deleted on process exit by calling cleanup
   class TempFileManager
+    SEC_IN_DAY=86400
+    # assume no transfer last longer than this
+    # (garbage collect file list which were not deleted after transfer)
+    FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
+    private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
     include Singleton
     def initialize
       @created_files=[]
@@ -33,5 +38,19 @@ module Aspera
       username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
       return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
     end
+    def cleanup_expired(temp_folder)
+      # garbage collect undeleted files
+      Dir.entries(temp_folder).each do |name|
+        file_path=File.join(temp_folder,name)
+        age_sec=(Time.now - File.stat(file_path).mtime).to_i
+        # check age of file, delete too old
+        if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
+          Log.log.debug("garbage collecting #{name}")
+          File.delete(file_path)
+        end
+      end
+    end
   end
 end