aspera-cli 4.0.0.pre1

data/lib/aspera/proxy_auto_config.rb

@@ -0,0 +1,34 @@
+require 'uri'
+require 'resolv'
+require 'erb'
+
+module Aspera
+  # evaluate a proxy autoconfig script
+  class ProxyAutoConfig
+    # template file is read once, it contains functions that can be used in a proxy autoconf script
+    PAC_FUNC_TEMPLATE=File.read(__FILE__.gsub(/\.rb$/,'.erb.js'))
+    private_constant :PAC_FUNC_TEMPLATE
+    # @param proxy_auto_config the proxy auto config script to be evaluated
+    def initialize(proxy_auto_config)
+      @proxy_auto_config=proxy_auto_config
+    end
+
+    # execut proxy auto config script for the given URL
+    def get_proxy(service_url)
+      # require at runtime, in case there is no js engine
+      require 'execjs'
+      # variables starting with "context_" are replaced in the ERB template file
+      # I did not find an easy way for the javascript to callback ruby
+      # and anyway, it only needs to get DNS translation
+      context_self='127.0.0.1'
+      context_host=URI.parse(service_url).host
+      context_ip=nil
+      Resolv::DNS.open{|dns|dns.each_address(context_host){|r_addr|context_ip=r_addr.to_s if r_addr.is_a?(Resolv::IPv4)}}
+      raise "DNS name not found: #{context_host}" if context_ip.nil?
+      pac_functions=ERB.new(PAC_FUNC_TEMPLATE).result(binding)
+      context = ExecJS.compile(pac_functions+@proxy_auto_config)
+      return context.call("FindProxyForURL", service_url, context_host)
+    end
+  end
+end
+

data/lib/aspera/rest.rb

@@ -0,0 +1,296 @@
+require 'aspera/log'
+require 'aspera/oauth'
+require 'aspera/rest_error_analyzer'
+require 'aspera/hash_ext'
+require 'aspera/rest_errors_aspera'
+require 'net/http'
+require 'net/https'
+require 'json'
+require 'base64'
+
+# add cancel method to http
+class Net::HTTP::Cancel < Net::HTTPRequest
+  METHOD = 'CANCEL'
+  REQUEST_HAS_BODY  = false
+  RESPONSE_HAS_BODY = false
+end
+
+#class Net::HTTP::Delete < Net::HTTPRequest
+#  METHOD = 'DELETE'
+#  REQUEST_HAS_BODY  = false
+#  RESPONSE_HAS_BODY = false
+#end
+
+module Aspera
+  # a simple class to make HTTP calls, equivalent to rest-client
+  # rest call errors are raised as exception RestCallError
+  # and error are analyzed in RestErrorAnalyzer
+  class Rest
+    private
+    # create and start keep alive connection on demand
+    def http_session
+      if @http_session.nil?
+        uri=self.class.build_uri(@params[:base_url])
+        # this honors http_proxy env var
+        @http_session=Net::HTTP.new(uri.host, uri.port)
+        @http_session.use_ssl = uri.scheme.eql?('https')
+        Log.log.debug("insecure=#{@@insecure}")
+        @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if @@insecure
+        @http_session.set_debug_output($stdout) if @@debug
+        if @params.has_key?(:session_cb)
+          @params[:session_cb].call(@http_session)
+        end
+        # manually start session for keep alive (if supported by server, else, session is closed every time)
+        @http_session.start
+      end
+      return @http_session
+    end
+
+    # set to true enables debug in HTTP class
+    @@debug=false
+    # true if https ignore certificate
+    @@insecure=false
+    @@user_agent='Ruby'
+
+    public
+
+    def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
+
+    def self.insecure; @@insecure;end
+
+    def self.user_agent=(v); @@user_agent=v;Log.log.debug("user_agent => #{@@user_agent}".red);end
+
+    def self.user_agent; @@user_agent;end
+
+    def self.debug=(flag); @@debug=flag; Log.log.debug("debug http => #{flag}"); end
+
+    def self.basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
+
+    attr_reader :params
+
+    # @param a_rest_params default call parameters and authentication (:auth) :
+    # :type (:basic, :oauth2, :url)
+    # :username   [:basic]
+    # :password   [:basic]
+    # :url_creds  [:url]
+    # :session_cb a lambda which takes @http_session as arg, use this to change parameters
+    # :*          [:oauth2] see Oauth class
+    def initialize(a_rest_params)
+      raise "ERROR: expecting Hash" unless a_rest_params.is_a?(Hash)
+      raise "ERROR: expecting base_url" unless a_rest_params[:base_url].is_a?(String)
+      @params=a_rest_params.clone
+      Log.dump('REST params',@params)
+      # base url without trailing slashes (note: string may be frozen)
+      @params[:base_url]=@params[:base_url].gsub(/\/+$/,'')
+      @http_session=nil
+      # default is no auth
+      @params[:auth]||={:type=>:none}
+      @params[:not_auth_codes]||=['401']
+      # translate old auth parameters, remove prefix, place in auth
+      [:auth,:basic,:oauth].each do |p_sym|
+        p_str=p_sym.to_s+'_'
+        @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
+          name=k_sym.to_s[p_str.length..-1]
+          name='grant' if k_sym.eql?(:oauth_type)
+          @params[:auth][name.to_sym]=@params[k_sym]
+          @params.delete(k_sym)
+        end
+      end
+      @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
+      Log.dump('REST params(2)',@params)
+    end
+
+    def oauth_token(options={})
+      raise "ERROR: not Oauth" unless @oauth.is_a?(Oauth)
+      return @oauth.get_authorization(options)
+    end
+
+    # build URI from URL and parameters and check it is http or https
+    def self.build_uri(url,params=nil)
+      uri=URI.parse(url)
+      raise "REST endpoint shall be http(s)" unless ['http','https'].include?(uri.scheme)
+      if !params.nil?
+        # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
+        if params.is_a?(Hash)
+          orig=params
+          params=[]
+          orig.each do |k,v|
+            case v
+            when Array
+              suffix=v.first.eql?('[]') ? v.shift : ''
+              v.each do |e|
+                params.push([k+suffix,e])
+              end
+            else
+              params.push([k,v])
+            end
+          end
+        end
+        # CGI.unescape to transform back %5D into []
+        uri.query=CGI.unescape(URI.encode_www_form(params))
+      end
+      return uri
+    end
+
+    # HTTP/S REST call
+    # call_data has keys:
+    # :auth
+    # :operation
+    # :subpath
+    # :headers
+    # :json_params
+    # :url_params
+    # :www_body_params
+    # :text_body_params
+    # :save_to_file (filepath)
+    # :return_error (bool)
+    def call(call_data)
+      raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
+      Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
+      call_data[:headers]||={}
+      call_data[:headers]['User-Agent'] ||= @@user_agent
+      call_data=@params.deep_merge(call_data)
+      case call_data[:auth][:type]
+      when :none
+        # no auth
+      when :basic
+        Log.log.debug("using Basic auth")
+        basic_auth_data=[call_data[:auth][:username],call_data[:auth][:password]]
+      when :oauth2
+        call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
+      when :url
+        call_data[:url_params]||={}
+        call_data[:auth][:url_creds].each do |key, value|
+          call_data[:url_params][key]=value
+        end
+      else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
+      end
+      # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
+      # URI.escape()
+      uri=self.class.build_uri("#{@params[:base_url]}/#{call_data[:subpath]}",call_data[:url_params])
+      Log.log.debug("URI=#{uri}")
+      begin
+        # instanciate request object based on string name
+        req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri.request_uri)
+      rescue NameError => e
+        raise "unsupported operation : #{call_data[:operation]}"
+      end
+      if call_data.has_key?(:json_params) and !call_data[:json_params].nil? then
+        req.body=JSON.generate(call_data[:json_params])
+        Log.dump('body JSON data',call_data[:json_params])
+        #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
+        req['Content-Type'] = 'application/json'
+        #call_data[:headers]['Accept']='application/json'
+      end
+      if call_data.has_key?(:www_body_params) then
+        req.body=URI.encode_www_form(call_data[:www_body_params])
+        Log.log.debug("body www data=#{req.body.chomp}")
+        req['Content-Type'] = 'application/x-www-form-urlencoded'
+      end
+      if call_data.has_key?(:text_body_params) then
+        req.body=call_data[:text_body_params]
+        Log.log.debug("body data=#{req.body.chomp}")
+      end
+      # set headers
+      if call_data.has_key?(:headers) then
+        call_data[:headers].keys.each do |key|
+          req[key] = call_data[:headers][key]
+        end
+      end
+      # :type = :basic
+      req.basic_auth(*basic_auth_data) unless basic_auth_data.nil?
+
+      Log.log.debug("call_data = #{call_data}")
+      result={:http=>nil}
+      begin
+        # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
+        oauth_tries ||= 2
+        Log.log.debug("send request")
+        http_session.request(req) do |response|
+          result[:http] = response
+          if call_data.has_key?(:save_to_file)
+            require 'ruby-progressbar'
+            total_size=result[:http]['Content-Length'].to_i
+            progress=ProgressBar.create(
+            :format     => '%a %B %p%% %r KB/sec %e',
+            :rate_scale => lambda{|rate|rate/1024},
+            :title      => 'progress',
+            :total      => total_size)
+            Log.log.debug("before write file")
+            target_file=call_data[:save_to_file]
+            if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
+              target_file=m[1]
+            end
+            Log.log.debug("saving to: #{target_file}")
+            File.open(target_file, "wb") do |file|
+              result[:http].read_body do |fragment|
+                file.write(fragment)
+                new_process=progress.progress+fragment.length
+                new_process = total_size if new_process > total_size
+                progress.progress=new_process
+              end
+            end
+            progress=nil
+          end
+        end
+        # sometimes there is a ITF8 char (e.g. (c) )
+        result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
+        Log.log.debug("result: body=#{result[:http].body}")
+        result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
+        case result_mime
+        when 'application/json','application/vnd.api+json'
+          result[:data]=JSON.parse(result[:http].body) rescue nil
+        else #when 'text/plain'
+          result[:data]=result[:http].body
+        end
+        Log.dump("result: parsed: #{result_mime}",result[:data])
+        Log.log.debug("result: code=#{result[:http].code}")
+        RestErrorAnalyzer.instance.raiseOnError(req,result)
+      rescue RestCallError => e
+        # not authorized: oauth token expired
+        if @params[:not_auth_codes].include?(result[:http].code.to_s) and call_data[:auth][:type].eql?(:oauth2)
+          begin
+            # try to use refresh token
+            req['Authorization']=oauth_token(refresh: true)
+          rescue RestCallError => e
+            Log.log.error("refresh failed".bg_red)
+            # regenerate a brand new token
+            req['Authorization']=oauth_token
+          end
+          Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
+          retry unless (oauth_tries -= 1).zero?
+        end # if
+        # raise exception if could not retry and not return error in result
+        raise e unless call_data[:return_error]
+      end
+      Log.log.debug("result=#{result}")
+      return result
+
+    end
+
+    #
+    # CRUD methods here
+    #
+
+    # @param encoding : one of: :json_params, :url_params
+    def create(subpath,params,encoding=:json_params)
+      return call({:operation=>'POST',:subpath=>subpath,:headers=>{'Accept'=>'application/json'},encoding=>params})
+    end
+
+    def read(subpath,args=nil)
+      return call({:operation=>'GET',:subpath=>subpath,:headers=>{'Accept'=>'application/json'},:url_params=>args})
+    end
+
+    def update(subpath,params)
+      return call({:operation=>'PUT',:subpath=>subpath,:headers=>{'Accept'=>'application/json'},:json_params=>params})
+    end
+
+    def delete(subpath)
+      return call({:operation=>'DELETE',:subpath=>subpath,:headers=>{'Accept'=>'application/json'}})
+    end
+
+    def cancel(subpath)
+      return call({:operation=>'CANCEL',:subpath=>subpath,:headers=>{'Accept'=>'application/json'}})
+    end
+  end
+end #module Aspera

data/lib/aspera/rest_call_error.rb

@@ -0,0 +1,13 @@
+module Aspera
+  # raised on error after REST call
+  class RestCallError < StandardError
+    attr_accessor :request
+    attr_accessor :response
+    # @param http response
+    def initialize(req,resp,msg)
+      @request = req
+      @response = resp
+      super(msg)
+    end
+  end
+end

data/lib/aspera/rest_error_analyzer.rb

@@ -0,0 +1,98 @@
+require 'aspera/log'
+require 'aspera/rest_call_error'
+require 'singleton'
+
+module Aspera
+  # analyze error codes returned by REST calls and raise ruby exception
+  class RestErrorAnalyzer
+    include Singleton
+    # the singleton object is registered with application specific handlers
+    def initialize
+      # list of handlers
+      @error_handlers=[]
+      self.add_handler('Type Generic') do |type,context|
+        if !context[:response].code.start_with?('2')
+          # add generic information
+          RestErrorAnalyzer.add_error(context,type,"#{context[:request]['host']} #{context[:response].code} #{context[:response].message}")
+        end
+      end
+    end
+
+    # Use this method to analyze a EST result and raise an exception
+    # Analyzes REST call response and raises a RestCallError exception
+    # if HTTP result code is not 2XX
+    def raiseOnError(req,res)
+      context={
+        messages: [],
+        request:  req,
+        response: res[:http],
+        data:     res[:data]
+      }
+      # multiple error messages can be found
+      # analyze errors from provided handlers
+      # note that there can be an error even if code is 2XX
+      @error_handlers.each do |handler|
+        begin
+          #Log.log.debug("test exception: #{handler[:name]}")
+          handler[:block].call(handler[:name],context)
+        rescue => e
+          Log.log.error("ERROR in handler:\n#{e.message}\n#{e.backtrace}")
+        end
+      end
+      unless context[:messages].empty?
+        raise RestCallError.new(context[:request],context[:response],context[:messages].join("\n"))
+      end
+    end
+
+    # add a new error handler (done at application initialisation)
+    # @param name : name of error handler (for logs)
+    # @param block : processing of response: takes two parameters: name, context
+    # name is the one provided here
+    # context is built in method raiseOnError
+    def add_handler(name,&block)
+      @error_handlers.unshift({name: name, block: block})
+    end
+
+    # add a simple error handler
+    # check that key exists and is string under specified path (hash)
+    # adds other keys as secondary information
+    def add_simple_handler(name,*args)
+      add_handler(name) do |type,context|
+        # need to clone because we modify and same array is used subsequently
+        path=args.clone
+        #Log.log.debug("path=#{path}")
+        # if last in path is boolean it tells if the error is only with http error code or always
+        always=[true, false].include?(path.last) ? path.pop : false
+        if context[:data].is_a?(Hash) and (!context[:response].code.start_with?('2') or always)
+          msg_key=path.pop
+          # dig and find sub entry corresponding to path in deep hash
+          error_struct=path.inject(context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
+          if error_struct.is_a?(Hash) and error_struct[msg_key].is_a?(String)
+            RestErrorAnalyzer.add_error(context,type,error_struct[msg_key])
+            error_struct.each do |k,v|
+              next if k.eql?(msg_key)
+              RestErrorAnalyzer.add_error(context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
+            end
+          end
+        end
+      end
+    end # add_simple_handler
+
+    # used by handler to add an error description to list of errors
+    # for logging and tracing : collect error descriptions (create file to activate)
+    # @param context a Hash containing the result context, provided to handler
+    # @param type a string describing type of exception, for logging purpose
+    # @param msg one error message  to add to list
+    def self.add_error(context,type,msg)
+      context[:messages].push(msg)
+      # log error for further analysis (file must exist to activate)
+      exc_log_file=File.join(Fasp::Installation.instance.config_folder,'exceptions.log')
+      if File.exist?(exc_log_file)
+        File.open(exc_log_file,'a+') do |f|
+          f.write("\n=#{type}=====\n#{context[:request].method} #{context[:request].path}\n#{context[:response].code}\n#{JSON.generate(context[:data])}\n#{context[:messages].join("\n")}")
+        end
+      end
+    end
+
+  end
+end

data/lib/aspera/rest_errors_aspera.rb

@@ -0,0 +1,58 @@
+require 'aspera/rest_error_analyzer'
+require 'aspera/log'
+
+module Aspera
+  # REST error handlers for various Aspera REST APIs
+  class RestErrorsAspera
+    # handlers should probably be defined by plugins for modularity
+    def self.registerHandlers
+      Log.log.debug("registering Aspera REST error handlers")
+      # Faspex 4: both user_message and internal_message, and code 200
+      # example: missing meta data on package creation
+      RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
+      RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
+      RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
+      # AoC Automation
+      RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
+      RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
+      RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
+      RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,context|
+        if context[:data].is_a?(Hash) and context[:data]['errors'].is_a?(Hash)
+          context[:data]['errors'].each do |k,v|
+            RestErrorAnalyzer.add_error(context,name,"#{k}: #{v}")
+          end
+        end
+      end
+      # call to upload_setup and download_setup of node api
+      RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,context|
+        if context[:data].is_a?(Hash)
+          d_t_s=context[:data]['transfer_specs']
+          if d_t_s.is_a?(Array)
+            d_t_s.each do |res|
+              #r_err=res['transfer_spec']['error']
+              r_err=res['error']
+              if r_err.is_a?(Hash)
+                RestErrorAnalyzer.add_error(context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
+              end
+            end
+          end
+        end
+      end
+      RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
+      RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
+      RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,context|
+        if context[:data].is_a?(Hash)
+          d_t_s=context[:data]['errors']
+          if d_t_s.is_a?(Array)
+            d_t_s.each do |res|
+              r_err=res['message']
+              if r_err.is_a?(String)
+                RestErrorAnalyzer.add_error(context,type,r_err)
+              end
+            end
+          end
+        end
+      end
+    end # registerErrorTypes
+  end
+end