aspera-cli 4.4.0 → 4.7.0

data/lib/aspera/proxy_auto_config.rb

@@ -1,34 +1,125 @@
+# frozen_string_literal: true
 require 'uri'
 require 'resolv'
-require 'erb'
+
+module URI
+  class Generic
+    # save original method that finds proxy in URI::Generic, it uses env var http_proxy
+    alias :find_proxy_orig find_proxy
+    def self.register_proxy_finder(&block)
+      # overload the method
+      define_method(:find_proxy) {|envars=ENV| block.call(to_s) || find_proxy_orig(envars)}
+    end
+  end
+end
 
 module Aspera
-  # evaluate a proxy autoconfig script
+  # Evaluate a proxy autoconfig script
   class ProxyAutoConfig
     # template file is read once, it contains functions that can be used in a proxy autoconf script
-    PAC_FUNC_TEMPLATE=File.read(__FILE__.gsub(/\.rb$/,'.erb.js'))
-    private_constant :PAC_FUNC_TEMPLATE
-    # @param proxy_auto_config the proxy auto config script to be evaluated
-    def initialize(proxy_auto_config)
-      @proxy_auto_config=proxy_auto_config
-    end
+    # it is similar to mozilla ascii_pac_utils.inc
+    PAC_FUNCTIONS_FILE=__FILE__.gsub(/\.rb$/,'.js').freeze
+    PAC_MAIN_FUNCTION='FindProxyForURL'
+    private_constant :PAC_FUNCTIONS_FILE,:PAC_MAIN_FUNCTION
+
+    private
 
-    # execut proxy auto config script for the given URL
-    def get_proxy(service_url)
-      # require at runtime, in case there is no js engine
-      require 'execjs'
-      # variables starting with "context_" are replaced in the ERB template file
-      # I did not find an easy way for the javascript to callback ruby
-      # and anyway, it only needs to get DNS translation
+    # variables starting with "context_" are replaced in the ERB template file
+    # I did not find an easy way for the javascript to callback ruby
+    # and anyway, it only needs to get DNS translation
+    def pac_dns_functions(context_host)
       context_self='127.0.0.1'
-      context_host=URI.parse(service_url).host
       context_ip=nil
       Resolv::DNS.open{|dns|dns.each_address(context_host){|r_addr|context_ip=r_addr.to_s if r_addr.is_a?(Resolv::IPv4)}}
       raise "DNS name not found: #{context_host}" if context_ip.nil?
-      pac_functions=ERB.new(PAC_FUNC_TEMPLATE).result(binding)
-      context = ExecJS.compile(pac_functions+@proxy_auto_config)
-      return context.call("FindProxyForURL", service_url, context_host)
+      # NOTE: Javascript code here with string inclusions
+      javascript=<<END_OF_JAVASCRIPT
+      function dnsResolve(host) {
+        if (host == '#{context_host}' || host == '#{context_ip}')
+          return '#{context_ip}';
+        throw 'only DNS for initial host, not ' + host;
+      }
+      function myIpAddress() {
+        return '#{context_self}';
+      }
+END_OF_JAVASCRIPT
+      return javascript
+    end
+
+    public
+
+    # @param proxy_auto_config the proxy auto config script to be evaluated
+    def initialize(proxy_auto_config)
+      # user provided javascript with FindProxyForURL function
+      @proxy_auto_config=proxy_auto_config
+      # avoid multiple execution, this does not support load balancing
+      @cache={}
+      @pac_functions=nil
+    end
+
+    def register_uri_generic
+      URI::Generic.register_proxy_finder{|url_str|get_proxies(url_str).first}
+      # allow chaining
+      return self
+    end
+
+    # execute proxy auto config script for the given URL : https://en.wikipedia.org/wiki/Proxy_auto-config
+    # @return either nil, or a String formated following PAC standard
+    def find_proxy_for_url(service_url)
+      uri=URI.parse(service_url)
+      simple_url="#{uri.scheme}://#{uri.host}"
+      if !@cache.has_key?(simple_url)
+        Log.log.debug("PAC: starting javascript for #{service_url}")
+        # require at runtime, in case there is no js engine
+        require 'execjs'
+        # read template lib
+        @pac_functions=File.read(PAC_FUNCTIONS_FILE).freeze if @pac_functions.nil?
+        # to be executed is dns + utils + user function
+        js_to_execute="#{pac_dns_functions(uri.host)}#{@pac_functions}#{@proxy_auto_config}"
+        executable_js = ExecJS.compile(js_to_execute)
+        @cache[simple_url]=executable_js.call(PAC_MAIN_FUNCTION, simple_url, uri.host)
+        Log.log.debug("PAC: result: #{@cache[simple_url]}")
+      end
+      return @cache[simple_url]
+    end
+
+    # used to replace URI::Generic.find_proxy
+    # @return Array of URI, possibly empty
+    def get_proxies(service_url)
+      # prepare result
+      uri_list=[]
+      # execute PAC script
+      proxy_list_str=find_proxy_for_url(service_url)
+      if !proxy_list_str.is_a?(String)
+        Log.log.warn("PAC: did not return a String, returned #{proxy_list_str.class}")
+        return uri_list
+      end
+      proxy_list_str.strip!
+      proxy_list_str.gsub!(/\s+/,' ')
+      proxy_list_str.split(';').each do |item|
+        # strip and split by space
+        parts=item.strip.split
+        case parts.shift
+        when 'DIRECT'
+          raise 'DIRECT has no param' unless parts.empty?
+        when 'PROXY'
+          addr_port=parts.shift
+          raise 'PROXY shall have one param' unless addr_port.is_a?(String) && parts.empty?
+          begin
+            # PAC proxy addresses are <host>:<port>
+            if addr_port.match(/:[0-9]+$/)
+              # we want to return URIs, so add dummy scheme
+              uri_list.push(URI.parse("proxy://#{addr_port}"))
+            else
+              Log.log.warn("PAC: PROXY must be <address>:<port>, ignoring #{addr_port}")
+            end
+          rescue StandardError
+            Log.log.warn("PAC: cannot parse #{addr_port}")
+          end
+        else Log.log.warn("PAC: ignoring proxy type #{parts.first}: not supported")
+        end
+      end
+      return uri_list
     end
   end
 end
-

data/lib/aspera/rest.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'aspera/log'
 require 'aspera/oauth'
 require 'aspera/rest_error_analyzer'
@@ -28,52 +29,52 @@ module Aspera
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    private
-    # set to true enables debug in HTTP class
-    @@debug=false
-    # true if https ignore certificate
-    @@insecure=false
-    @@user_agent='Ruby'
-    @@download_partial_suffix='.http_partial'
-
-    public
-    def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
-
-    def self.insecure; @@insecure;end
-
-    def self.user_agent=(v); @@user_agent=v;Log.log.debug("user_agent => #{@@user_agent}".red);end
+    # global settings also valid for any subclass
+    @@global={ # rubocop:disable Style/ClassVars
+      debug: false,
+      # true if https ignore certificate
+      insecure: false,
+      user_agent: 'Ruby',
+      download_partial_suffix: '.http_partial',
+      # a lambda which takes the Net::HTTP as arg, use this to change parameters
+      session_cb: nil
+    }
 
-    def self.user_agent; @@user_agent;end
-
-    def self.debug=(flag); @@debug=flag; Log.log.debug("debug http => #{flag}"); end
+    class<<self
+      # define accessors
+      @@global.keys.each do |p|
+        define_method(p){@@global[p]}
+        define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p]=val}
+      end
 
-    def self.basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
+      def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
 
-    # build URI from URL and parameters and check it is http or https
-    def self.build_uri(url,params=nil)
-      uri=URI.parse(url)
-      raise "REST endpoint shall be http(s)" unless ['http','https'].include?(uri.scheme)
-      if !params.nil?
-        # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
-        if params.is_a?(Hash)
-          orig=params
-          params=[]
-          orig.each do |k,v|
-            case v
-            when Array
-              suffix=v.first.eql?('[]') ? v.shift : ''
-              v.each do |e|
-                params.push([k+suffix,e])
+      # build URI from URL and parameters and check it is http or https
+      def build_uri(url,params=nil)
+        uri=URI.parse(url)
+        raise 'REST endpoint shall be http(s)' unless ['http','https'].include?(uri.scheme)
+        if !params.nil?
+          # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
+          if params.is_a?(Hash)
+            orig=params
+            params=[]
+            orig.each do |k,v|
+              case v
+              when Array
+                suffix=v.first.eql?('[]') ? v.shift : ''
+                v.each do |e|
+                  params.push([k+suffix,e])
+                end
+              else
+                params.push([k,v])
               end
-            else
-              params.push([k,v])
             end
           end
+          # CGI.unescape to transform back %5D into []
+          uri.query=CGI.unescape(URI.encode_www_form(params))
         end
-        # CGI.unescape to transform back %5D into []
-        uri.query=CGI.unescape(URI.encode_www_form(params))
+        return uri
       end
-      return uri
     end
 
     private
@@ -85,12 +86,10 @@ module Aspera
         # this honors http_proxy env var
         @http_session=Net::HTTP.new(uri.host, uri.port)
         @http_session.use_ssl = uri.scheme.eql?('https')
-        Log.log.debug("insecure=#{@@insecure}")
-        @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if @@insecure
-        @http_session.set_debug_output($stdout) if @@debug
-        if @params.has_key?(:session_cb)
-          @params[:session_cb].call(@http_session)
-        end
+        @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if self.class.insecure
+        @http_session.set_debug_output($stdout) if self.class.debug
+        # set http options in callback, such as timeout and cert. verification
+        self.class.session_cb.call(@http_session) unless self.class.session_cb.nil?
         # manually start session for keep alive (if supported by server, else, session is closed every time)
         @http_session.start
       end
@@ -100,33 +99,72 @@ module Aspera
     public
 
     attr_reader :params
-    attr_reader :oauth
 
-    # @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
-    # :type (:basic, :oauth2, :url, :none)
-    # :username   [:basic]
-    # :password   [:basic]
-    # :url_creds  [:url]
-    # :session_cb a lambda which takes @http_session as arg, use this to change parameters
-    # :*          [:oauth2] see Oauth class
+    def oauth
+      if @oauth.nil?
+        raise 'ERROR: no OAuth defined' unless @params[:auth][:type].eql?(:oauth2)
+        @oauth=Oauth.new(@params[:auth])
+      end
+      return @oauth
+    end
+
+    # @param a_rest_params [Hash] default call parameters (merged at call)
     def initialize(a_rest_params)
-      raise "ERROR: expecting Hash" unless a_rest_params.is_a?(Hash)
-      raise "ERROR: expecting base_url" unless a_rest_params[:base_url].is_a?(String)
+      raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
+      raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
       @params=a_rest_params.clone
       Log.dump('REST params',@params)
       # base url without trailing slashes (note: string may be frozen)
       @params[:base_url]=@params[:base_url].gsub(/\/+$/,'')
       @http_session=nil
       # default is no auth
-      @params[:auth]||={:type=>:none}
+      @params[:auth]||={type: :none}
       @params[:not_auth_codes]||=['401']
-      @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
+      @oauth=nil
       Log.dump('REST params(2)',@params)
     end
 
-    def oauth_token(options={})
-      raise "ERROR: expecting Oauth, have #{@oauth.class}" unless @oauth.is_a?(Oauth)
-      return @oauth.get_authorization(options)
+    def oauth_token(force_refresh: false)
+      raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
+      return oauth.get_authorization(use_refresh_token: force_refresh)
+    end
+
+    def build_request(call_data)
+      # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
+      # URI.escape()
+      uri=self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
+      Log.log.debug("URI=#{uri}")
+      begin
+        # instanciate request object based on string name
+        req=Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
+      rescue NameError
+        raise "unsupported operation : #{call_data[:operation]}"
+      end
+      if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
+        req.body=JSON.generate(call_data[:json_params])
+        Log.dump('body JSON data',call_data[:json_params])
+        #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
+        req['Content-Type'] = 'application/json'
+        #call_data[:headers]['Accept']='application/json'
+      end
+      if call_data.has_key?(:www_body_params)
+        req.body=URI.encode_www_form(call_data[:www_body_params])
+        Log.log.debug("body www data=#{req.body.chomp}")
+        req['Content-Type'] = 'application/x-www-form-urlencoded'
+      end
+      if call_data.has_key?(:text_body_params)
+        req.body=call_data[:text_body_params]
+        Log.log.debug("body data=#{req.body.chomp}")
+      end
+      # set headers
+      if call_data.has_key?(:headers)
+        call_data[:headers].keys.each do |key|
+          req[key] = call_data[:headers][key]
+        end
+      end
+      # :type = :basic
+      req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
+      return req
     end
 
     # HTTP/S REST call
@@ -139,21 +177,31 @@ module Aspera
     # :url_params
     # :www_body_params
     # :text_body_params
-    # :save_to_file (filepath)
-    # :return_error (bool)
+    # :save_to_file (filepath) default: nil
+    # :return_error (bool) default: nil
+    # :redirect_max (int) default: 0
+    # :not_auth_codes (array)
+    # ----
+    # authentication (:auth) :
+    # :type (:none, :basic, :oauth2, :url)
+    # :username   [:basic]
+    # :password   [:basic]
+    # :url_creds  [:url] a hash
+    # :*          [:oauth2] see Oauth class
     def call(call_data)
       raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
+      call_data[:subpath]='' if call_data[:subpath].nil?
       Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
       call_data[:headers]||={}
-      call_data[:headers]['User-Agent'] ||= @@user_agent
-      # defaults from @params are overriden by call dataz
+      call_data[:headers]['User-Agent'] ||= self.class.user_agent
+      # defaults from @params are overriden by call data
       call_data=@params.deep_merge(call_data)
       case call_data[:auth][:type]
       when :none
         # no auth
       when :basic
-        Log.log.debug("using Basic auth")
-        basic_auth_data=[call_data[:auth][:username],call_data[:auth][:password]]
+        Log.log.debug('using Basic auth')
+        # done in build_req
       when :oauth2
         call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
       when :url
@@ -163,67 +211,33 @@ module Aspera
         end
       else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
       end
-      # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
-      # URI.escape()
-      uri=self.class.build_uri("#{@params[:base_url]}#{call_data[:subpath].nil? ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
-      Log.log.debug("URI=#{uri}")
-      begin
-        # instanciate request object based on string name
-        req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri.request_uri)
-      rescue NameError => e
-        raise "unsupported operation : #{call_data[:operation]}"
-      end
-      if call_data.has_key?(:json_params) and !call_data[:json_params].nil? then
-        req.body=JSON.generate(call_data[:json_params])
-        Log.dump('body JSON data',call_data[:json_params])
-        #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
-        req['Content-Type'] = 'application/json'
-        #call_data[:headers]['Accept']='application/json'
-      end
-      if call_data.has_key?(:www_body_params) then
-        req.body=URI.encode_www_form(call_data[:www_body_params])
-        Log.log.debug("body www data=#{req.body.chomp}")
-        req['Content-Type'] = 'application/x-www-form-urlencoded'
-      end
-      if call_data.has_key?(:text_body_params) then
-        req.body=call_data[:text_body_params]
-        Log.log.debug("body data=#{req.body.chomp}")
-      end
-      # set headers
-      if call_data.has_key?(:headers) then
-        call_data[:headers].keys.each do |key|
-          req[key] = call_data[:headers][key]
-        end
-      end
-      # :type = :basic
-      req.basic_auth(*basic_auth_data) unless basic_auth_data.nil?
-
+      req=build_request(call_data)
       Log.log.debug("call_data = #{call_data}")
-      result={:http=>nil}
+      result={http: nil}
       # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
-        tries_remain_redirect||=4
-        Log.log.debug("send request")
+        tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
+        Log.log.debug('send request')
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
-          if call_data.has_key?(:save_to_file)
+          if !call_data[:save_to_file].nil? && result[:http].code.to_s.start_with?('2')
             total_size=result[:http]['Content-Length'].to_i
             progress=ProgressBar.create(
-            :format     => '%a %B %p%% %r KB/sec %e',
-            :rate_scale => lambda{|rate|rate/1024},
-            :title      => 'progress',
-            :total      => total_size)
-            Log.log.debug("before write file")
+            format:     '%a %B %p%% %r KB/sec %e',
+            rate_scale: lambda{|rate|rate/1024},
+            title:      'progress',
+            total:      total_size)
+            Log.log.debug('before write file')
             target_file=call_data[:save_to_file]
             # override user's path to path in header
-            if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
+            if !response['Content-Disposition'].nil? && (m=response['Content-Disposition'].match(/filename="([^"]+)"/))
               target_file=File.join(File.dirname(target_file),m[1])
             end
             # download with temp filename
-            target_file_tmp="#{target_file}#{@@download_partial_suffix}"
+            target_file_tmp="#{target_file}#{self.class.download_partial_suffix}"
             Log.log.debug("saving to: #{target_file}")
             File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
@@ -238,27 +252,27 @@ module Aspera
             progress=nil
           end # save_to_file
         end
-        # sometimes there is a ITF8 char (e.g. (c) )
-        result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
+        # sometimes there is a UTF8 char (e.g. (c) )
+        result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
         Log.log.debug("result: body=#{result[:http].body}")
         result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
-        case result_mime
+        result[:data]=case result_mime
         when 'application/json','application/vnd.api+json'
-          result[:data]=JSON.parse(result[:http].body) rescue nil
+          JSON.parse(result[:http].body) rescue nil
         else #when 'text/plain'
-          result[:data]=result[:http].body
+          result[:http].body
         end
         Log.dump("result: parsed: #{result_mime}",result[:data])
         Log.log.debug("result: code=#{result[:http].code}")
-        RestErrorAnalyzer.instance.raiseOnError(req,result)
+        RestErrorAnalyzer.instance.raise_on_error(req,result)
       rescue RestCallError => e
         # not authorized: oauth token expired
-        if @params[:not_auth_codes].include?(result[:http].code.to_s) and call_data[:auth][:type].eql?(:oauth2)
+        if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
           begin
             # try to use refresh token
-            req['Authorization']=oauth_token(refresh: true)
+            req['Authorization']=oauth_token(force_refresh: true)
           rescue RestCallError => e
-            Log.log.error("refresh failed".bg_red)
+            Log.log.error('refresh failed'.bg_red)
             # regenerate a brand new token
             req['Authorization']=oauth_token
           end
@@ -266,25 +280,30 @@ module Aspera
           retry unless (oauth_tries -= 1).zero?
         end # if oauth
         # moved ?
-        if e.response.is_a?(Net::HTTPRedirection)
-          if tries_remain_redirect > 0
-            tries_remain_redirect-=1
-            Log.log.info("URL is moved: #{e.response['location']}")
-            raise e
-            # TODO: rebuild request with new location
-            #retry
+        raise e unless e.response.is_a?(Net::HTTPRedirection)
+        if tries_remain_redirect.positive?
+          tries_remain_redirect-=1
+          Log.log.info("URL is moved: #{e.response['location']}")
+          current_uri=URI.parse(call_data[:base_url])
+          redir_uri=URI.parse(e.response['location'])
+          call_data[:base_url]=e.response['location']
+          call_data[:subpath]=''
+          if current_uri.host.eql?(redir_uri.host) && current_uri.port.eql?(redir_uri.port)
+            req=build_request(call_data)
+            retry
           else
-            raise "too many redirect"
+            # change host
+            Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
+            return self.class.new(call_data).call(call_data)
           end
         else
-          raise e
+          raise e unless call_data[:return_error]
         end
         # raise exception if could not retry and not return error in result
         raise e unless call_data[:return_error]
       end # begin request
       Log.log.debug("result=#{result}")
       return result
-
     end
 
     #
@@ -293,23 +312,23 @@ module Aspera
 
     # @param encoding : one of: :json_params, :url_params
     def create(subpath,params,encoding=:json_params)
-      return call({:operation=>'POST',:subpath=>subpath,:headers=>{'Accept'=>'application/json'},encoding=>params})
+      return call({operation: 'POST',subpath: subpath,headers: {'Accept'=>'application/json'},encoding=>params})
     end
 
     def read(subpath,args=nil)
-      return call({:operation=>'GET',:subpath=>subpath,:headers=>{'Accept'=>'application/json'},:url_params=>args})
+      return call({operation: 'GET',subpath: subpath,headers: {'Accept'=>'application/json'},url_params: args})
     end
 
     def update(subpath,params)
-      return call({:operation=>'PUT',:subpath=>subpath,:headers=>{'Accept'=>'application/json'},:json_params=>params})
+      return call({operation: 'PUT',subpath: subpath,headers: {'Accept'=>'application/json'},json_params: params})
     end
 
     def delete(subpath)
-      return call({:operation=>'DELETE',:subpath=>subpath,:headers=>{'Accept'=>'application/json'}})
+      return call({operation: 'DELETE',subpath: subpath,headers: {'Accept'=>'application/json'}})
     end
 
     def cancel(subpath)
-      return call({:operation=>'CANCEL',:subpath=>subpath,:headers=>{'Accept'=>'application/json'}})
+      return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept'=>'application/json'}})
     end
   end
 end #module Aspera

data/lib/aspera/rest_call_error.rb

@@ -1,8 +1,8 @@
+# frozen_string_literal: true
 module Aspera
   # raised on error after REST call
   class RestCallError < StandardError
-    attr_accessor :request
-    attr_accessor :response
+    attr_accessor :request, :response
     # @param http response
     def initialize(req,resp,msg)
       @request = req