aspera-cli 4.4.0 → 4.5.0

data/lib/aspera/faspex_gw.rb

@@ -1,6 +1,6 @@
 require 'aspera/log'
 require 'aspera/aoc'
-require 'aspera/node'
+require 'aspera/fasp/default'
 require 'aspera/cli/main'
 require 'webrick'
 require 'webrick/https'
@@ -76,9 +76,9 @@ module Aspera
         faspex_transfer_spec={
           'direction'   => 'send',
           'remote_host' => node_info['host'],
-          'remote_user' => Node::ACCESS_KEY_TRANSFER_USER,
-          'ssh_port'    => Node::SSH_PORT_DEFAULT,
-          'fasp_port'   => Node::UDP_PORT_DEFAULT
+          'remote_user' => Fasp::Default::ACCESS_KEY_TRANSFER_USER,
+          'ssh_port'    => Fasp::Default::SSH_PORT,
+          'fasp_port'   => Fasp::Default::UDP_PORT,
           'tags'        => ts_tags,
           'token'       => node_auth_bearer_token,
           'paths'       => [{'destination' => '/'}],
@@ -95,8 +95,8 @@ module Aspera
           'lock_rate_policy' => true,
           'source_root' => '',
           'content_protection' => nil,
-          'target_rate_cap_kbps' => 20000, # TODO
-          'target_rate_kbps' => 10000, # TODO
+          'target_rate_cap_kbps' => 20000, # TODO: is this value useful ?
+          'target_rate_kbps' => 10000, # TODO: get from where?
           'cipher' => 'aes-128',
           'cipher_allowed' => nil,
           'http_fallback' => false,

data/lib/aspera/keychain/encrypted_hash.rb

@@ -0,0 +1,120 @@
+require 'openssl'
+
+module Aspera
+  module Keychain
+    class SimpleCipher
+      def initialize(key)
+        @key=Digest::SHA1.hexdigest(key)[0..23]
+        @cipher = OpenSSL::Cipher.new('DES-EDE3-CBC')
+      end
+
+      def encrypt(value)
+        @cipher.encrypt
+        @cipher.key = @key
+        s = @cipher.update(value) + @cipher.final
+        s.unpack('H*').first
+      end
+
+      def decrypt(value)
+        @cipher.decrypt
+        @cipher.key = @key
+        s = [value].pack('H*').unpack('C*').pack('c*')
+        @cipher.update(s) + @cipher.final
+      end
+    end
+
+    # Manage secrets in a simple Hash
+    class EncryptedHash
+      SEPARATOR='%'
+      private_constant :SEPARATOR
+      def initialize(values)
+        raise "values shall be Hash" unless values.is_a?(Hash)
+        @all_secrets=values
+      end
+
+      def set(options)
+        raise "options shall be Hash" unless options.is_a?(Hash)
+        unsupported=options.keys-[:username,:url,:secret,:description]
+        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        username=options[:username]
+        raise "options shall have username" if username.nil?
+        url=options[:url]
+        raise "options shall have username" if url.nil?
+        secret=options[:secret]
+        raise "options shall have secret" if secret.nil?
+        key=[url,username].join(SEPARATOR)
+        raise "secret #{key} already exist, delete first" if @all_secrets.has_key?(key)
+        obj={username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
+        obj[:description]=options[:description] if options.has_key?(:description)
+        @all_secrets[key]=obj
+        nil
+      end
+
+      def list
+        result=[]
+        @all_secrets.each do |k,v|
+          case v
+          when String
+            o={username: k, url: '', description: ''}
+          when Hash
+            o=v.clone
+            o.delete(:secret)
+            o[:description]||=''
+          else raise "error"
+          end
+          o[:description]=v[:description] if v.is_a?(Hash) and v[:description].is_a?(String)
+          result.push(o)
+        end
+        return result
+      end
+
+      def delete(options)
+        raise "options shall be Hash" unless options.is_a?(Hash)
+        unsupported=options.keys-[:username,:url]
+        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        username=options[:username]
+        raise "options shall have username" if username.nil?
+        url=options[:url]
+        key=nil
+        if !url.nil?
+          extk=[url,username].join(SEPARATOR)
+          key=extk if @all_secrets.has_key?(extk)
+        end
+        # backward compatibility: TODO: remove in future ? (make url mandatory ?)
+        key=username if key.nil? and @all_secrets.has_key?(username)
+        raise "no such secret" if key.nil?
+        @all_secrets.delete(key)
+      end
+
+      def get(options)
+        raise "options shall be Hash" unless options.is_a?(Hash)
+        unsupported=options.keys-[:username,:url]
+        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        username=options[:username]
+        raise "options shall have username" if username.nil?
+        url=options[:url]
+        val=nil
+        if !url.nil?
+          val=@all_secrets[[url,username].join(SEPARATOR)]
+        end
+        # backward compatibility: TODO: remove in future ? (make url mandatory ?)
+        if val.nil?
+          val=@all_secrets[username]
+        end
+        result=options.clone
+        case val
+        when NilClass
+          raise "no such secret"
+        when String
+          result.merge!({secret: val, description: ''})
+        when Hash
+          key=[url,username].join(SEPARATOR)
+          plain=SimpleCipher.new(key).decrypt(val[:secret])
+          result.merge!({secret: plain, description: val[:description]})
+        else raise "error"
+        end
+        return result
+      end
+    end
+  end
+end

data/lib/aspera/keychain/macos_security.rb

@@ -0,0 +1,94 @@
+require 'security'
+
+# enhance the gem to support other keychains
+module Security
+  class Keychain
+    class << self
+      def by_name(name)
+        keychains_from_output(`security list-keychains`).select{|kc|kc.filename.end_with?("/#{name}.keychain-db")}.first
+      end
+    end
+  end
+
+  class Password
+    class << self
+      # add some login to original method
+      alias_method :orig_flags_for_options, :flags_for_options
+      def flags_for_options(options = {})
+        keychain=options.delete(:keychain)
+        url=options.delete(:url)
+        if !url.nil?
+          uri=URI.parse(url)
+          raise 'only https' unless uri.scheme.eql?('https')
+          options[:r]='htps'
+          raise 'host required in URL' if uri.host.nil?
+          options[:s]=uri.host
+          options[:p]=uri.path unless ['','/'].include?(uri.path)
+          options[:P]=uri.port unless uri.port.eql?(443) and !url.include?(':443/')
+        end
+        flags=[orig_flags_for_options(options)]
+        flags.push(keychain.filename) unless keychain.nil?
+        flags.join(' ')
+      end
+    end
+  end
+end
+
+module Aspera
+  module Keychain
+    # keychain based on macOS keychain, using `security` cmmand line
+    class MacosSecurity
+      def initialize(name=nil)
+        if name.nil?
+          @keychain=Security::Keychain.default_keychain
+        else
+          @keychain=Security::Keychain.by_name(name)
+        end
+        raise "no such keychain #{name}" if @keychain.nil?
+      end
+
+      def set(options)
+        raise 'options shall be Hash' unless options.is_a?(Hash)
+        unsupported=options.keys-[:username,:url,:secret,:description]
+        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        username=options[:username]
+        raise 'options shall have username' if username.nil?
+        url=options[:url]
+        raise 'options shall have url' if url.nil?
+        secret=options[:secret]
+        raise 'options shall have secret' if secret.nil?
+        raise 'set not implemented'
+        self
+      end
+
+      def get(options)
+        raise 'options shall be Hash' unless options.is_a?(Hash)
+        unsupported=options.keys-[:username,:url]
+        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        username=options[:username]
+        raise 'options shall have username' if username.nil?
+        url=options[:url]
+        raise 'options shall have url' if url.nil?
+        info=Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
+        raise 'not found' if info.nil?
+        result=options.clone
+        result.merge!({secret: info.password, description: info.attributes['icmt']})
+        return result
+      end
+
+      def list
+        raise 'list not implemented'
+      end
+
+      def delete(options)
+        raise 'options shall be Hash' unless options.is_a?(Hash)
+        unsupported=options.keys-[:username,:url]
+        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        username=options[:username]
+        raise 'options shall have username' if username.nil?
+        url=options[:url]
+        raise 'delete not implemented'
+      end
+    end
+  end
+end

data/lib/aspera/log.rb

@@ -7,33 +7,41 @@ require 'singleton'
 module Aspera
   # Singleton object for logging
   class Log
-
-    public
     include Singleton
+    # class methods
+    class << self
+      # levels are :debug,:info,:warn,:error,fatal,:unknown
+      def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
-    attr_reader :logger
-    attr_reader :logger_type
-    # levels are :debug,:info,:warn,:error,fatal,:unknown
-    def self.levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
-
-    # where logs are sent to
-    def self.logtypes; [:stderr,:stdout,:syslog];end
+      # where logs are sent to
+      def logtypes; [:stderr,:stdout,:syslog];end
 
-    # get the logger object of singleton
-    def self.log; self.instance.logger; end
+      # get the logger object of singleton
+      def log; instance.logger;end
 
-    # dump object in debug mode
-    # @param name string or symbol
-    # @param format either pp or json format
-    def self.dump(name,object,format=:json)
-      result=case format
-      when :ruby;PP.pp(object,'')
-      when :json;JSON.pretty_generate(object) rescue PP.pp(object,'')
-      else raise "wrong parameter, expect pp or json"
+      # dump object in debug mode
+      # @param name string or symbol
+      # @param format either pp or json format
+      def dump(name,object,format=:json)
+        self.log.debug() do
+          result=case format
+          when :json
+            JSON.pretty_generate(object) rescue PP.pp(object,'')
+          when :ruby
+            PP.pp(object,'')
+          else
+            raise "wrong parameter, expect pp or json"
+          end
+          "#{name.to_s.green} (#{format})=\n#{result}"
+        end
       end
-      self.log.debug("#{name.to_s.green} (#{format})=\n#{result}")
     end
 
+    attr_reader :logger_type
+    attr_reader :logger
+    attr_writer :program_name
+    attr_accessor :log_passwords
+
     # set log level of underlying logger given symbol level
     def level=(new_level)
       @logger.level=Logger::Severity.const_get(new_level.to_sym.upcase)
@@ -44,20 +52,15 @@ module Aspera
       Logger::Severity.constants.each do |name|
         return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
       end
+      # should not happen
       raise "error"
     end
 
     # change underlying logger, but keep log level
     def logger_type=(new_logtype)
-      current_severity_integer=if @logger.nil?
-        if ENV.has_key?('AS_LOG_LEVEL')
-          ENV['AS_LOG_LEVEL']
-        else
-          Logger::Severity::WARN
-        end
-      else
-        @logger.level
-      end
+      current_severity_integer=@logger.level unless @logger.nil?
+      current_severity_integer=ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? and ENV.has_key?('AS_LOG_LEVEL')
+      current_severity_integer=Logger::Severity::WARN if current_severity_integer.nil?
       case new_logtype
       when :stderr
         @logger = Logger.new(STDERR)
@@ -71,17 +74,27 @@ module Aspera
       end
       @logger.level=current_severity_integer
       @logger_type=new_logtype
+      original_formatter = @logger.formatter || Logger::Formatter.new
+      # update formatter with password hiding
+      @logger.formatter=proc do |severity, datetime, progname, msg|
+        unless @log_passwords
+          msg=msg.gsub(/("[^"]*(password|secret|private_key)[^"]*"=>")([^"]+)(")/){"#{$1}***#{$4}"}
+          msg=msg.gsub(/("[^"]*(secret)[^"]*"=>{)([^}]+)(})/){"#{$1}***#{$4}"}
+          msg=msg.gsub(/((secrets)={)([^}]+)(})/){"#{$1}***#{$4}"}
+        end
+        original_formatter.call(severity, datetime, progname, msg)
+      end
     end
 
-    attr_writer :program_name
-
     private
 
     def initialize
       @logger=nil
       @program_name='aspera'
-      # this sets @logger and @logger_type
+      @log_passwords=false
+      # this sets @logger and @logger_type (self needed to call method instead of local var)
       self.logger_type=:stderr
+      raise "error logger shall be defined" if @logger.nil?
     end
 
   end

data/lib/aspera/node.rb

@@ -1,3 +1,4 @@
+require 'aspera/fasp/default'
 require 'aspera/rest'
 require 'aspera/oauth'
 require 'aspera/log'
@@ -10,16 +11,12 @@ module Aspera
     # permissions
     ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
     MATCH_EXEC_PREFIX='exec:'
-    # (public) default transfer username for access key based transfers
-    ACCESS_KEY_TRANSFER_USER='xfer'
-    SSH_PORT_DEFAULT=33001
-    UDP_PORT_DEFAULT=33001
 
     # register node special token decoder
     Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
 
     def self.set_ak_basic_token(ts,ak,secret)
-      raise "ERROR: expected xfer" unless ts['remote_user'].eql?(ACCESS_KEY_TRANSFER_USER)
+      raise "ERROR: expected xfer" unless ts['remote_user'].eql?(Fasp::Default::ACCESS_KEY_TRANSFER_USER)
       ts['token']="Basic #{Base64.strict_encode64("#{ak}:#{secret}")}"
     end
 
@@ -30,7 +27,7 @@ module Aspera
     def self.file_matcher(match_expression)
       match_expression||="#{MATCH_EXEC_PREFIX}true"
       if match_expression.start_with?(MATCH_EXEC_PREFIX)
-        return eval "lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
+        return eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
       end
       return lambda{|f|f['name'].match(/#{match_expression}/)}
     end

data/lib/aspera/rest.rb

@@ -35,8 +35,14 @@ module Aspera
     @@insecure=false
     @@user_agent='Ruby'
     @@download_partial_suffix='.http_partial'
+    # a lambda which takes the Net::HTTP as arg, use this to change parameters
+    @@session_cb=nil
 
     public
+    def self.session_cb=(v); @@session_cb=v;Log.log.debug("session_cb => #{@@session_cb}".red);end
+
+    def self.session_cb; @@session_cb;end
+
     def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
 
     def self.insecure; @@insecure;end
@@ -85,12 +91,9 @@ module Aspera
         # this honors http_proxy env var
         @http_session=Net::HTTP.new(uri.host, uri.port)
         @http_session.use_ssl = uri.scheme.eql?('https')
-        Log.log.debug("insecure=#{@@insecure}")
         @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if @@insecure
         @http_session.set_debug_output($stdout) if @@debug
-        if @params.has_key?(:session_cb)
-          @params[:session_cb].call(@http_session)
-        end
+        @@session_cb.call(@http_session) unless @@session_cb.nil?
         # manually start session for keep alive (if supported by server, else, session is closed every time)
         @http_session.start
       end
@@ -107,7 +110,6 @@ module Aspera
     # :username   [:basic]
     # :password   [:basic]
     # :url_creds  [:url]
-    # :session_cb a lambda which takes @http_session as arg, use this to change parameters
     # :*          [:oauth2] see Oauth class
     def initialize(a_rest_params)
       raise "ERROR: expecting Hash" unless a_rest_params.is_a?(Hash)
@@ -129,47 +131,14 @@ module Aspera
       return @oauth.get_authorization(options)
     end
 
-    # HTTP/S REST call
-    # call_data has keys:
-    # :auth
-    # :operation
-    # :subpath
-    # :headers
-    # :json_params
-    # :url_params
-    # :www_body_params
-    # :text_body_params
-    # :save_to_file (filepath)
-    # :return_error (bool)
-    def call(call_data)
-      raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
-      Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
-      call_data[:headers]||={}
-      call_data[:headers]['User-Agent'] ||= @@user_agent
-      # defaults from @params are overriden by call dataz
-      call_data=@params.deep_merge(call_data)
-      case call_data[:auth][:type]
-      when :none
-        # no auth
-      when :basic
-        Log.log.debug("using Basic auth")
-        basic_auth_data=[call_data[:auth][:username],call_data[:auth][:password]]
-      when :oauth2
-        call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
-      when :url
-        call_data[:url_params]||={}
-        call_data[:auth][:url_creds].each do |key, value|
-          call_data[:url_params][key]=value
-        end
-      else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
-      end
+    def build_request(call_data)
       # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
       # URI.escape()
-      uri=self.class.build_uri("#{@params[:base_url]}#{call_data[:subpath].nil? ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
+      uri=self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
       Log.log.debug("URI=#{uri}")
       begin
         # instanciate request object based on string name
-        req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri.request_uri)
+        req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri)#.request_uri
       rescue NameError => e
         raise "unsupported operation : #{call_data[:operation]}"
       end
@@ -196,20 +165,59 @@ module Aspera
         end
       end
       # :type = :basic
-      req.basic_auth(*basic_auth_data) unless basic_auth_data.nil?
+      req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
+      return req
+    end
 
+    # HTTP/S REST call
+    # call_data has keys:
+    # :auth
+    # :operation
+    # :subpath
+    # :headers
+    # :json_params
+    # :url_params
+    # :www_body_params
+    # :text_body_params
+    # :save_to_file (filepath)
+    # :return_error (bool)
+    # :redirect_max (int)
+    def call(call_data)
+      raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
+      call_data[:subpath]='' if call_data[:subpath].nil?
+      Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
+      call_data[:headers]||={}
+      call_data[:headers]['User-Agent'] ||= @@user_agent
+      # defaults from @params are overriden by call data
+      call_data=@params.deep_merge(call_data)
+      case call_data[:auth][:type]
+      when :none
+        # no auth
+      when :basic
+        Log.log.debug("using Basic auth")
+        # done in build_req
+      when :oauth2
+        call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
+      when :url
+        call_data[:url_params]||={}
+        call_data[:auth][:url_creds].each do |key, value|
+          call_data[:url_params][key]=value
+        end
+      else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
+      end
+      req=build_request(call_data)
       Log.log.debug("call_data = #{call_data}")
       result={:http=>nil}
       # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
-        tries_remain_redirect||=4
+        tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
         Log.log.debug("send request")
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
-          if call_data.has_key?(:save_to_file)
+          if call_data.has_key?(:save_to_file) and result[:http].code.to_s.start_with?('2')
             total_size=result[:http]['Content-Length'].to_i
             progress=ProgressBar.create(
             :format     => '%a %B %p%% %r KB/sec %e',
@@ -238,7 +246,7 @@ module Aspera
             progress=nil
           end # save_to_file
         end
-        # sometimes there is a ITF8 char (e.g. (c) )
+        # sometimes there is a UTF8 char (e.g. (c) )
         result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
         Log.log.debug("result: body=#{result[:http].body}")
         result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
@@ -270,9 +278,18 @@ module Aspera
           if tries_remain_redirect > 0
             tries_remain_redirect-=1
             Log.log.info("URL is moved: #{e.response['location']}")
-            raise e
-            # TODO: rebuild request with new location
-            #retry
+            current_uri=URI.parse(call_data[:base_url])
+            redir_uri=URI.parse(e.response['location'])
+            call_data[:base_url]=e.response['location']
+            call_data[:subpath]=''
+            if current_uri.host.eql?(redir_uri.host) and current_uri.port.eql?(redir_uri.port)
+              req=build_request(call_data)
+              retry
+            else
+              # change host
+              Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
+              return self.class.new(call_data).call(call_data)
+            end
           else
             raise "too many redirect"
           end
@@ -284,7 +301,6 @@ module Aspera
       end # begin request
       Log.log.debug("result=#{result}")
       return result
-
     end
 
     #