aspera-cli 4.17.0 → 4.18.1

data/lib/aspera/oauth/web.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'aspera/oauth/base'
-require 'aspera/open_application'
+require 'aspera/environment'
 require 'aspera/web_auth'
 require 'aspera/assert'
 module Aspera
@@ -34,7 +34,7 @@ module Aspera
         # start a web server to receive request code
         web_server = WebAuth.new(@redirect_uri)
         # start browser on login page
-        OpenApplication.instance.uri(login_page_url)
+        Environment.instance.open_uri(login_page_url)
         # wait for code in request
         received_params = web_server.received_request
         Aspera.assert(random_state.eql?(received_params['state'])){'wrong received state'}

data/lib/aspera/persistency_folder.rb

@@ -74,5 +74,5 @@ module Aspera
       return File.join(@folder, "#{object_id}#{FILE_SUFFIX}")
       # .gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
     end
-  end # PersistencyFolder
-end # Aspera
+  end
+end

data/lib/aspera/preview/generator.rb

@@ -252,6 +252,6 @@ module Aspera
           '+repage',
           @destination_file_path])
       end
-    end # Generator
-  end # Preview
-end # Aspera
+    end
+  end
+end

data/lib/aspera/preview/options.rb

@@ -36,6 +36,6 @@ module Aspera
       DESCRIPTIONS.each do |opt|
         attr_accessor opt[:name]
       end
-    end # Options
-  end # Preview
-end # Aspera
+    end
+  end
+end

data/lib/aspera/preview/terminal.rb

@@ -95,7 +95,7 @@ module Aspera
           end
           false
         end
-      end # class << self
-    end # class Terminal
-  end # module Preview
-end # module Aspera
+      end
+    end
+  end
+end

data/lib/aspera/preview/utils.rb

@@ -117,6 +117,6 @@ module Aspera
           return output_file
         end
       end
-    end # Options
-  end # Preview
-end # Aspera
+    end
+  end
+end

data/lib/aspera/proxy_auto_config.rb

@@ -36,7 +36,11 @@ module Aspera
     def pac_dns_functions(context_host)
       context_self = '127.0.0.1'
       context_ip = nil
-      Resolv::DNS.open{|dns|dns.each_address(context_host){|r_addr|context_ip = r_addr.to_s if r_addr.is_a?(Resolv::IPv4)}}
+      Resolv::DNS.open do |dns|
+        dns.each_address(context_host) do |r_addr|
+          context_ip = r_addr.to_s if r_addr.is_a?(Resolv::IPv4)
+        end
+      end
       raise "DNS name not found: #{context_host}" if context_ip.nil?
       # NOTE: Javascript code here with string inclusions
       javascript = <<END_OF_JAVASCRIPT

data/lib/aspera/rest.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
+require 'aspera/rest_errors_aspera'
+require 'aspera/rest_error_analyzer'
 require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/oauth'
-require 'aspera/rest_error_analyzer'
 require 'aspera/hash_ext'
-require 'aspera/rest_errors_aspera'
 require 'net/http'
 require 'net/https'
 require 'json'
 require 'base64'
 require 'cgi'
 
-# add cancel method to http
+# Cancel method for HTTP
 class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
   METHOD = 'CANCEL'
   REQUEST_HAS_BODY  = false
@@ -24,12 +24,16 @@ module Aspera
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # global settings also valid for any subclass
+    # Global settings also valid for any subclass
+    # @param user_agent [String] HTTP request header: 'User-Agent'
+    # @param download_partial_suffix [String] suffix for partial download
+    # @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
+    # @param progress_bar [Object] progress bar object
     @@global = { # rubocop:disable Style/ClassVars
-      user_agent:              'Ruby',          # goes to HTTP request header: 'User-Agent'
-      download_partial_suffix: '.http_partial', # suffix for partial download
-      session_cb:              nil,             # a lambda which takes the Net::HTTP as arg, use this to change parameters
-      progress_bar:            nil # progress bar object
+      user_agent:              'RubyAsperaRest',
+      download_partial_suffix: '.http_partial',
+      session_cb:              nil,
+      progress_bar:            nil
     }
 
     # flag for array parameters prefixed with []
@@ -44,9 +48,10 @@ module Aspera
     JSON_DECODE = ['application/json', 'application/vnd.api+json', 'application/x-javascript'].freeze
 
     class << self
+      # @return [String] Basic auth token
       def basic_token(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
 
-      # used to build a parameter list prefixed with "[]"
+      # Build a parameter list prefixed with "[]"
       # @param values [Array] list of values
       def array_params(values)
         return [ARRAY_PARAMS].concat(values)
@@ -56,7 +61,7 @@ module Aspera
         return values.first.eql?(ARRAY_PARAMS)
       end
 
-      # build URI from URL and parameters and check it is http or https, encode array [] parameters
+      # Build URI from URL and parameters and check it is http or https, encode array [] parameters
       def build_uri(url, query_hash=nil)
         uri = URI.parse(url)
         Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
@@ -82,8 +87,16 @@ module Aspera
         return uri
       end
 
+      # decode query string as hash
+      # Does not support arrays in query string, no standard, e.g. PHP's way is p[]=1&p[]=2
+      # @param query [String] query string
+      # @return [Hash] decoded query
       def decode_query(query)
-        URI.decode_www_form(query).each_with_object({}){|v, h|h[v.first] = v.last }
+        URI.decode_www_form(query).each_with_object({}) do |pair, h|
+          key = pair.first
+          raise "Array not supported in query string: #{key}" if key.include?('[]') || h.key?(key)
+          h[key] = pair.last
+        end
       end
 
       # Start a HTTP/S session, also used for web sockets
@@ -141,6 +154,17 @@ module Aspera
       def user_agent
         return @@global[:user_agent]
       end
+
+      def parse_header(header)
+        type, *params = header.split(/;\s*/)
+        parameters = params.map do |param|
+          one = param.split(/=\s*/)
+          one[0] = one[0].to_sym
+          one[1] = one[1].gsub(/\A"|"\z/, '')
+          one
+        end.to_h
+        { type: type.downcase, parameters: parameters }
+      end
     end
 
     private
@@ -153,47 +177,6 @@ module Aspera
       return @http_session
     end
 
-    def build_request(
-      operation:,
-      subpath:,
-      url_params:,
-      json_params:,
-      www_body_params:,
-      text_body_params:,
-      headers:
-    )
-      # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
-      # URI.escape()
-      separator = !['', '/'].include?(subpath) || @base_url.end_with?('/') ? '/' : ''
-      uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", url_params)
-      Log.log.debug{"URI=#{uri}"}
-      begin
-        # instantiate request object based on string name
-        req = Net::HTTP.const_get(operation.capitalize).new(uri)
-      rescue NameError
-        raise "unsupported operation : #{operation}"
-      end
-      if !json_params.nil?
-        req.body = JSON.generate(json_params) # , ascii_only: true
-        req['Content-Type'] = 'application/json'
-      end
-      if !www_body_params.nil?
-        req.body = URI.encode_www_form(www_body_params)
-        req['Content-Type'] = 'application/x-www-form-urlencoded'
-      end
-      if !text_body_params.nil?
-        req.body = text_body_params
-      end
-      # set headers
-      headers.each do |key, value|
-        req[key] = value
-      end
-      # :type = :basic
-      req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
-      Log.log.debug{Log.dump(:req_body, req.body)}
-      return req
-    end
-
     public
 
     attr_reader :auth_params
@@ -217,7 +200,7 @@ module Aspera
     #     :url_query  [:url]    a hash
     #     :*          [:oauth2] see OAuth::Factory class
     # @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
-    # @param redirect_max [int] max redirections allowed
+    # @param redirect_max [int] max redirection allowed
     def initialize(
       base_url:,
       auth: nil,
@@ -226,8 +209,12 @@ module Aspera
       headers: nil
     )
       Aspera.assert_type(base_url, String)
-      # base url with max one trailing slashes (note: string may be frozen)
-      @base_url = base_url.gsub(%r{//+$}, '/')
+      # base url with no trailing slashes (note: string may be frozen)
+      @base_url = base_url.gsub(%r{/+$}, '')
+      # remove trailing port if it is 443 and scheme is https
+      @base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
+      @base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
+      Log.log.debug{"Rest.new(#{@base_url})"}
       # default is no auth
       @auth_params = auth.nil? ? {type: :none} : auth
       Aspera.assert_type(@auth_params, Hash)
@@ -255,21 +242,21 @@ module Aspera
       return @oauth
     end
 
-    def oauth_token(force_refresh: false)
-      Aspera.assert_values(force_refresh, [true, false])
-      return oauth.get_authorization(use_refresh_token: force_refresh)
-    end
-
     # HTTP/S REST call
+    # @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
+    # @param subpath [String] subpath of REST API
+    # @param query [Hash] URL parameters
+    # @param body [Hash, String] body parameters
+    # @param body_type [Symbol] type of body parameters (:json, :www, :text, nil)
     # @param save_to_file (filepath)
     # @param return_error (bool)
+    # @param headers [Hash] additional headers
     def call(
       operation:,
       subpath: nil,
-      json_params: nil,
-      url_params: nil,
-      www_body_params: nil,
-      text_body_params: nil,
+      query: nil,
+      body: nil,
+      body_type: nil,
       save_to_file: nil,
       return_error: false,
       headers: nil
@@ -293,20 +280,49 @@ module Aspera
         Log.log.debug('using Basic auth')
         # done in build_req
       when :oauth2
-        headers['Authorization'] = oauth_token unless headers.key?('Authorization')
+        headers['Authorization'] = oauth.token unless headers.key?('Authorization')
       when :url
-        url_params ||= {}
+        query ||= {}
         @auth_params[:url_query].each do |key, value|
-          url_params[key] = value
+          query[key] = value
         end
       else Aspera.error_unexpected_value(@auth_params[:type])
       end
       result = {http: nil}
-      # start a block to be able to retry the actual HTTP request
+      # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
       begin
-        req = build_request(
-          operation: operation, subpath: subpath, json_params: json_params, url_params: url_params, www_body_params: www_body_params,
-          text_body_params: text_body_params, headers: headers)
+        # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
+        # URI.escape()
+        separator = ['', '/'].include?(subpath) ? '' : '/'
+        uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
+        Log.log.debug{"URI=#{uri}"}
+        begin
+          # instantiate request object based on string name
+          req = Net::HTTP.const_get(operation.capitalize).new(uri)
+        rescue NameError
+          raise "unsupported operation : #{operation}"
+        end
+        case body_type
+        when :json
+          req.body = JSON.generate(body) # , ascii_only: true
+          req['Content-Type'] = 'application/json'
+        when :www
+          req.body = URI.encode_www_form(body)
+          req['Content-Type'] = 'application/x-www-form-urlencoded'
+        when :text
+          req.body = body
+          req['Content-Type'] = 'text/plain'
+        when nil
+        else
+          raise "unsupported body type : #{body_type}"
+        end
+        # set headers
+        headers.each do |key, value|
+          req[key] = value
+        end
+        # :type = :basic
+        req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
+        Log.log.debug{Log.dump(:req_body, req.body)}
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
         # initialize with number of initial retries allowed, nil gives zero
@@ -317,7 +333,7 @@ module Aspera
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
-          result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first.downcase
+          result_mime = self.class.parse_header(result[:http]['Content-Type'] || 'text/plain')[:type]
           # JSON data needs to be parsed, in case it contains an error code
           if !save_to_file.nil? &&
               result[:http].code.to_s.start_with?('2') &&
@@ -327,8 +343,11 @@ module Aspera
             Log.log.debug('before write file')
             target_file = save_to_file
             # override user's path to path in header
-            if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
-              target_file = File.join(File.dirname(target_file), m[1])
+            if !response['Content-Disposition'].nil?
+              disposition = self.class.parse_header(response['Content-Disposition'])
+              if disposition[:parameters].key?(:filename)
+                target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
+              end
             end
             # download with temp filename
             target_file_tmp = "#{target_file}#{@@global[:download_partial_suffix]}"
@@ -347,7 +366,7 @@ module Aspera
             # rename at the end
             File.rename(target_file_tmp, target_file)
             file_saved = true
-          end # save_to_file
+          end
         end
         # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
         # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
@@ -367,16 +386,16 @@ module Aspera
         if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
           begin
             # try to use refresh token
-            req['Authorization'] = oauth_token(force_refresh: true)
+            req['Authorization'] = oauth.token(refresh: true)
           rescue RestCallError => e_tok
             e = e_tok
             Log.log.error('refresh failed'.bg_red)
             # regenerate a brand new token
-            req['Authorization'] = oauth_token(force_refresh: true)
+            req['Authorization'] = oauth.token(refresh: true)
           end
           Log.log.debug{"using new token=#{headers['Authorization']}"}
           retry if (oauth_tries -= 1).nonzero?
-        end # if oauth
+        end
         # redirect ? (any code beginning with 3)
         if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
           tries_remain_redirect -= 1
@@ -390,13 +409,12 @@ module Aspera
           end
           # forwards the request to the new location
           return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
-            operation: operation, json_params: json_params,
-            url_params: url_params, www_body_params: www_body_params, text_body_params: text_body_params,
+            operation: operation, query: query, body: body, body_type: body_type,
             save_to_file: save_to_file, return_error: return_error, headers: headers)
         end
         # raise exception if could not retry and not return error in result
         raise e unless return_error
-      end # begin request
+      end
       Log.log.debug{"result=#{result}"}
       return result
     end
@@ -405,21 +423,20 @@ module Aspera
     # CRUD methods here
     #
 
-    # @param encoding : one of: :json_params, :url_params
-    def create(subpath, params, encoding=:json_params)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params)
+    def create(subpath, params)
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
     end
 
     def read(subpath, query=nil)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: query)
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)
     end
 
     def update(subpath, params)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params)
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
     end
 
     def delete(subpath, params=nil)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: params)
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)
     end
 
     def cancel(subpath)
@@ -453,4 +470,4 @@ module Aspera
       end
     end
   end
-end # module Aspera
+end

data/lib/aspera/rest_call_error.rb

@@ -3,7 +3,7 @@
 module Aspera
   # raised on error after REST call
   class RestCallError < StandardError
-    attr_accessor :request, :response
+    attr_reader :request, :response
 
     # @param req HTTP Request object
     # @param resp HTTP Response object

data/lib/aspera/rest_error_analyzer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'aspera/log'
 require 'aspera/rest_call_error'
+require 'aspera/log'
 require 'singleton'
 
 module Aspera
@@ -80,7 +80,7 @@ module Aspera
           end
         end
       end
-    end # add_simple_handler
+    end
 
     class << self
       # used by handler to add an error description to list of errors

data/lib/aspera/rest_errors_aspera.rb

@@ -61,7 +61,7 @@ module Aspera
             RestErrorAnalyzer.add_error(call_context, type, "missing parameter: #{param}")
           end
         end
-      end # register_handlers
+      end
     end
   end
 end

data/lib/aspera/resumer.rb

@@ -71,7 +71,7 @@ module Aspera
         sleep_seconds *= @parameters[:sleep_factor]
         # cap value
         sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
-      end # loop
+      end
     end
   end
 end

data/lib/aspera/secret_hider.rb

@@ -25,10 +25,8 @@ module Aspera
       /(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
       # rendered JSON or Ruby
       /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
-      # option "secret"
-      /(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
-      # option "secrets"
-      /(?<begin>(secrets)={)[^}]+(?<end>})/,
+      # logged data
+      /(?<begin>(?:#{ALL_SECRETS.join('|')})[ =:]+).*(?<end>$)/,
       # private key values
       /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
       # cred in http dump

data/lib/aspera/ssh.rb

@@ -61,7 +61,7 @@ module Aspera
         ssh_channel.wait
         # main ssh session loop
         session.loop
-      end # start
+      end
       # response as single string
       return response.join
     end